From 51471cfc0be394733307d1bd5c99484c305d0bf3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 01:45:25 +0000 Subject: [PATCH 1/8] feat(eval): canonicalize workspace mode/path configuration --- README.md | 3 +- apps/cli/src/commands/eval/commands/run.ts | 47 +--- apps/cli/src/commands/eval/run-eval.ts | 85 ++----- packages/core/src/evaluation/orchestrator.ts | 29 ++- .../core/src/evaluation/providers/targets.ts | 5 + packages/core/src/evaluation/types.ts | 6 +- .../evaluation/validation/eval-file.schema.ts | 19 +- packages/core/src/evaluation/yaml-parser.ts | 29 ++- .../core/test/evaluation/orchestrator.test.ts | 39 +++ .../test/evaluation/providers/targets.test.ts | 238 +----------------- .../evaluation/repo-schema-validation.test.ts | 42 ++++ .../workspace-config-parsing.test.ts | 58 +++++ 12 files changed, 226 insertions(+), 374 deletions(-) diff --git a/README.md b/README.md index 61b827c3e..81ee7cc70 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,6 @@ targets: - name: vscode_dev provider: vscode - workspace_template: ${{ WORKSPACE_PATH }} judge_target: azure-base - name: local_agent @@ -358,6 +357,8 @@ targets: Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`. +Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`). + Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields. ## Evaluation Features diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 5a9f3ca60..91dd72a68 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -105,53 +105,16 @@ export const evalRunCommand = command({ long: 'verbose', description: 'Enable verbose logging', }), - keepWorkspaces: flag({ - long: 'keep-workspaces', - description: - 'Always keep temporary workspaces after evaluation (default: keep on failure only)', - }), - cleanupWorkspaces: flag({ - long: 'cleanup-workspaces', - description: 'Always cleanup temporary workspaces, even on failure', - }), - poolWorkspaces: flag({ - long: 'pool-workspaces', - description: 'Enable workspace pooling (default for shared workspaces with repos)', - }), - noPool: flag({ - long: 'no-pool', - description: 'Disable workspace pooling (clone fresh each run)', - }), - workspace: option({ - type: optional(string), - long: 'workspace', - description: 'Use an existing directory as the workspace directly (skips clone/copy/pool)', - }), workspaceMode: option({ type: optional(string), long: 'workspace-mode', - description: "Workspace mode: 'pooled', 'ephemeral', or 'static'", + description: "Workspace mode: 'pooled', 'temp', or 'static'", }), workspacePath: option({ type: optional(string), long: 'workspace-path', description: 'Static workspace directory path (used when workspace mode is static)', }), - workspaceClean: option({ - type: optional(string), - long: 'workspace-clean', - description: "Pooled reset clean mode: 'standard' or 'full'", - }), - retainOnSuccess: option({ - type: optional(string), - long: 'retain-on-success', - description: "Workspace retention on success: 'keep' or 'cleanup'", - }), - retainOnFailure: option({ - type: optional(string), - long: 'retain-on-failure', - description: "Workspace retention on failure: 'keep' or 'cleanup'", - }), otelFile: option({ type: optional(string), long: 'otel-file', @@ -216,16 +179,8 @@ export const evalRunCommand = command({ cache: args.cache, noCache: args.noCache, verbose: args.verbose, - keepWorkspaces: args.keepWorkspaces, - cleanupWorkspaces: args.cleanupWorkspaces, - poolWorkspaces: args.poolWorkspaces, - noPool: args.noPool, - workspace: args.workspace, workspaceMode: args.workspaceMode, workspacePath: args.workspacePath, - workspaceClean: args.workspaceClean, - retainOnSuccess: args.retainOnSuccess, - retainOnFailure: args.retainOnFailure, trace: false, otelFile: args.otelFile, traceFile: args.traceFile, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 4a1fa753f..d887f5fcd 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -69,8 +69,6 @@ interface NormalizedOptions { readonly cache: boolean; readonly noCache: boolean; readonly verbose: boolean; - readonly keepWorkspaces: boolean; - readonly cleanupWorkspaces: boolean; readonly otelFile?: string; readonly traceFile?: string; readonly exportOtel: boolean; @@ -78,14 +76,8 @@ interface NormalizedOptions { readonly otelCaptureContent: boolean; readonly otelGroupTurns: boolean; readonly retryErrors?: string; - readonly poolWorkspaces?: boolean; - readonly poolMaxSlots?: number; - readonly workspace?: string; - readonly workspaceMode?: 'pooled' | 'ephemeral' | 'static'; + readonly workspaceMode?: 'pooled' | 'temp' | 'static'; readonly workspacePath?: string; - readonly workspaceClean?: 'standard' | 'full'; - readonly retainOnSuccess?: 'keep' | 'cleanup'; - readonly retainOnFailure?: 'keep' | 'cleanup'; } function normalizeBoolean(value: unknown): boolean { @@ -134,16 +126,8 @@ function normalizeOptionalNumber(value: unknown): number | undefined { return undefined; } -function normalizeWorkspaceMode(value: unknown): 'pooled' | 'ephemeral' | 'static' | undefined { - return value === 'pooled' || value === 'ephemeral' || value === 'static' ? value : undefined; -} - -function normalizeWorkspaceClean(value: unknown): 'standard' | 'full' | undefined { - return value === 'standard' || value === 'full' ? value : undefined; -} - -function normalizeRetention(value: unknown): 'keep' | 'cleanup' | undefined { - return value === 'keep' || value === 'cleanup' ? value : undefined; +function normalizeWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined { + return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined; } function normalizeOptions( @@ -198,6 +182,18 @@ function normalizeOptions( // Output dir: CLI --out > config output.dir > auto-generated const cliOut = normalizeString(rawOptions.out); const configOut = config?.output?.dir; + const cliWorkspacePath = normalizeString(rawOptions.workspacePath); + const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode); + const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode); + if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== 'static') { + throw new Error('--workspace-path requires --workspace-mode=static (or omit --workspace-mode)'); + } + + const yamlExecutionRecord = yamlExecution as Record | undefined; + const yamlWorkspaceMode = normalizeWorkspaceMode(yamlExecutionRecord?.workspace_mode); + const yamlWorkspacePath = normalizeString(yamlExecutionRecord?.workspace_path); + const workspacePath = cliWorkspacePath ?? yamlWorkspacePath; + const workspaceMode = cliWorkspacePath ? 'static' : (cliWorkspaceMode ?? yamlWorkspaceMode); return { target: singleTarget, @@ -223,11 +219,6 @@ function normalizeOptions( normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true || config?.execution?.verbose === true, - keepWorkspaces: - normalizeBoolean(rawOptions.keepWorkspaces) || - yamlExecution?.keep_workspaces === true || - config?.execution?.keepWorkspaces === true, - cleanupWorkspaces: normalizeBoolean(rawOptions.cleanupWorkspaces), // Precedence: CLI > YAML config > TS config otelFile: normalizeString(rawOptions.otelFile) ?? @@ -250,23 +241,8 @@ function normalizeOptions( otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent), otelGroupTurns: normalizeBoolean(rawOptions.otelGroupTurns), retryErrors: normalizeString(rawOptions.retryErrors), - // Pool: --no-pool explicitly disables; --pool-workspaces explicitly enables; YAML config; default undefined (orchestrator defaults to true) - poolWorkspaces: normalizeBoolean(rawOptions.noPool) - ? false - : normalizeBoolean(rawOptions.poolWorkspaces) - ? true - : yamlExecution?.pool_workspaces, - poolMaxSlots: yamlExecution?.pool_slots, - workspace: normalizeString(rawOptions.workspace), - workspaceMode: normalizeWorkspaceMode(rawOptions.workspaceMode), - workspacePath: normalizeString(rawOptions.workspacePath), - workspaceClean: normalizeWorkspaceClean(rawOptions.workspaceClean), - retainOnSuccess: - normalizeRetention(rawOptions.retainOnSuccess) ?? - (normalizeBoolean(rawOptions.keepWorkspaces) ? 'keep' : undefined), - retainOnFailure: - normalizeRetention(rawOptions.retainOnFailure) ?? - (normalizeBoolean(rawOptions.cleanupWorkspaces) ? 'cleanup' : undefined), + workspaceMode, + workspacePath, } satisfies NormalizedOptions; } @@ -578,7 +554,6 @@ async function runSingleEvalFile(params: { // Create streaming observer for real-time OTel span export const streamingObserver = otelExporter?.createStreamingObserver() ?? null; - const results = await evaluationRunner({ testFilePath, repoRoot, @@ -604,16 +579,8 @@ async function runSingleEvalFile(params: { evalCases, verbose: options.verbose, maxConcurrency: resolvedWorkers, - keepWorkspaces: options.keepWorkspaces, - cleanupWorkspaces: options.cleanupWorkspaces, - poolWorkspaces: options.poolWorkspaces, - poolMaxSlots: options.poolMaxSlots, - workspace: options.workspace, workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, - workspaceClean: options.workspaceClean, - retainOnSuccess: options.retainOnSuccess, - retainOnFailure: options.retainOnFailure, trials: trialsConfig, totalBudgetUsd, failOnError, @@ -717,30 +684,22 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise retryNonErrorResults = await loadNonErrorResults(retryPath); } - if (options.keepWorkspaces && options.cleanupWorkspaces) { - console.warn( - 'Warning: Both --keep-workspaces and --cleanup-workspaces specified. --cleanup-workspaces takes precedence.', - ); - } - // Validate static workspace path exists and is a directory - const explicitWorkspacePath = options.workspacePath ?? options.workspace; - if (explicitWorkspacePath) { - const resolvedWorkspace = path.resolve(explicitWorkspacePath); + if (options.workspacePath) { + const resolvedWorkspace = path.resolve(options.workspacePath); try { const { stat } = await import('node:fs/promises'); const stats = await stat(resolvedWorkspace); if (!stats.isDirectory()) { - throw new Error(`--workspace path is not a directory: ${resolvedWorkspace}`); + throw new Error(`--workspace-path is not a directory: ${resolvedWorkspace}`); } } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') { - throw new Error(`--workspace path does not exist: ${resolvedWorkspace}`); + throw new Error(`--workspace-path does not exist: ${resolvedWorkspace}`); } throw err; } - // Update workspace paths to resolved absolute path - options = { ...options, workspace: resolvedWorkspace, workspacePath: resolvedWorkspace }; + options = { ...options, workspacePath: resolvedWorkspace }; } if (options.verbose) { diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index f63c17be7..7cde86642 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -215,7 +215,7 @@ export interface RunEvaluationOptions { /** Pre-existing workspace directory to use directly (skips clone/copy/pool) */ readonly workspace?: string; /** Workspace materialization mode override */ - readonly workspaceMode?: 'pooled' | 'ephemeral' | 'static'; + readonly workspaceMode?: 'pooled' | 'temp' | 'static'; /** Static workspace path override (used when workspaceMode=static) */ readonly workspacePath?: string; /** Workspace clean policy override for pooled reset */ @@ -443,10 +443,15 @@ export async function runEvaluation( // Resolve worker count and pool mode const isPerTestIsolation = suiteWorkspace?.isolation === 'per_test'; - const configuredMode = suiteWorkspace?.mode ?? workspaceMode; - const configuredStaticPath = suiteWorkspace?.static_path ?? workspacePath ?? legacyWorkspacePath; - const useStaticWorkspace = - configuredMode === 'static' || (!!configuredStaticPath && !configuredMode); + const cliWorkspacePath = workspacePath ?? legacyWorkspacePath; + const yamlWorkspacePath = suiteWorkspace?.path; + if (cliWorkspacePath && workspaceMode && workspaceMode !== 'static') { + throw new Error('--workspace-path requires --workspace-mode static when both are provided'); + } + const configuredMode = + cliWorkspacePath ? 'static' : (workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? 'static' : 'pooled')); + const configuredStaticPath = cliWorkspacePath ?? yamlWorkspacePath; + const useStaticWorkspace = configuredMode === 'static'; // static workspace is incompatible with per_test isolation if (useStaticWorkspace && isPerTestIsolation) { @@ -455,7 +460,10 @@ export async function runEvaluation( ); } if (configuredMode === 'static' && !configuredStaticPath) { - throw new Error('workspace.mode=static requires workspace.static_path or --workspace-path'); + throw new Error('workspace.mode=static requires workspace.path or --workspace-path'); + } + if (configuredMode !== 'static' && configuredStaticPath) { + throw new Error('workspace.path requires workspace.mode=static'); } const hasSharedWorkspace = !!( @@ -465,13 +473,8 @@ export async function runEvaluation( (suiteWorkspace?.repos?.length && !isPerTestIsolation) ); - // Pool support: mode-based first, then legacy pool booleans, then default. - const poolEnabled = - configuredMode === 'pooled' - ? true - : configuredMode === 'ephemeral' || useStaticWorkspace - ? false - : (suiteWorkspace?.pool ?? poolWorkspaces ?? true); + // Pool support is mode-based: pooled enables, temp/static disable. + const poolEnabled = configuredMode === 'pooled'; const usePool = poolEnabled !== false && !!suiteWorkspace?.repos?.length && diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 15115d30d..1fb331d6c 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -696,6 +696,11 @@ export function resolveTargetDefinition( evalFilePath?: string, ): ResolvedTarget { const parsed = BASE_TARGET_SCHEMA.parse(definition); + if (parsed.workspace_template !== undefined || parsed.workspaceTemplate !== undefined) { + throw new Error( + `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`, + ); + } const provider = parsed.provider.toLowerCase(); const providerBatching = resolveOptionalBoolean( parsed.provider_batching ?? parsed.providerBatching, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 9c25d1219..95e1e49e8 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -277,11 +277,9 @@ export type WorkspaceConfig = { /** Workspace lifecycle hooks */ readonly hooks?: WorkspaceHooksConfig; /** Workspace materialization mode */ - readonly mode?: 'pooled' | 'ephemeral' | 'static'; + readonly mode?: 'pooled' | 'temp' | 'static'; /** Required when mode=static: use this existing directory directly */ - readonly static_path?: string; - /** @deprecated Use mode=pooled|ephemeral|static */ - readonly pool?: boolean; + readonly path?: string; }; export type CodeEvaluatorConfig = { diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 4d980e5bf..281ce8663 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -294,15 +294,16 @@ const WorkspaceHooksSchema = z.object({ after_all: WorkspaceHookSchema.optional(), }); -const WorkspaceSchema = z.object({ - template: z.string().optional(), - isolation: z.enum(['shared', 'per_test']).optional(), - repos: z.array(RepoSchema).optional(), - hooks: WorkspaceHooksSchema.optional(), - mode: z.enum(['pooled', 'ephemeral', 'static']).optional(), - static_path: z.string().optional(), - pool: z.boolean().optional(), -}); +const WorkspaceSchema = z + .object({ + template: z.string().optional(), + isolation: z.enum(['shared', 'per_test']).optional(), + repos: z.array(RepoSchema).optional(), + hooks: WorkspaceHooksSchema.optional(), + mode: z.enum(['pooled', 'temp', 'static']).optional(), + path: z.string().optional(), + }) + .strict(); // --------------------------------------------------------------------------- // Execution block diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 96cd47eea..5c9915654 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -676,6 +676,17 @@ async function resolveWorkspaceConfig( function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfig | undefined { if (!isJsonObject(raw)) return undefined; const obj = raw as Record; + if ('static_path' in obj) { + throw new Error( + "workspace.static_path has been removed. Use workspace.path with workspace.mode=static.", + ); + } + if ('pool' in obj) { + throw new Error("workspace.pool has been removed. Use workspace.mode='pooled' or 'temp'."); + } + if ('static' in obj) { + throw new Error("workspace.static has been removed. Use workspace.mode='static'."); + } let template = typeof obj.template === 'string' ? obj.template : undefined; if (template && !path.isAbsolute(template)) { @@ -692,14 +703,12 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi : undefined; const hooks = parseWorkspaceHooksConfig(obj.hooks, evalFileDir); - const mode = - obj.mode === 'pooled' || obj.mode === 'ephemeral' || obj.mode === 'static' - ? obj.mode - : undefined; - const staticPath = typeof obj.static_path === 'string' ? obj.static_path : undefined; - const pool = typeof obj.pool === 'boolean' ? obj.pool : undefined; + const explicitMode = + obj.mode === 'pooled' || obj.mode === 'temp' || obj.mode === 'static' ? obj.mode : undefined; + const workspacePath = typeof obj.path === 'string' ? obj.path : undefined; + const mode = explicitMode ?? (workspacePath ? 'static' : undefined); - if (!template && !isolation && !repos && !hooks && !mode && !staticPath && pool === undefined) + if (!template && !isolation && !repos && !hooks && !mode && !workspacePath) return undefined; return { @@ -708,8 +717,7 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi ...(repos !== undefined && { repos }), ...(hooks !== undefined && { hooks }), ...(mode !== undefined && { mode }), - ...(staticPath !== undefined && { static_path: staticPath }), - ...(pool !== undefined && { pool }), + ...(workspacePath !== undefined && { path: workspacePath }), }; } @@ -749,8 +757,7 @@ function mergeWorkspaceConfigs( repos: caseLevel.repos ?? suiteLevel.repos, ...(hasHooks && { hooks: mergedHooks as WorkspaceHooksConfig }), mode: caseLevel.mode ?? suiteLevel.mode, - static_path: caseLevel.static_path ?? suiteLevel.static_path, - pool: caseLevel.pool ?? suiteLevel.pool, + path: caseLevel.path ?? suiteLevel.path, }; } diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index f48a8b711..20be90450 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -2648,6 +2648,45 @@ describe('--workspace flag', () => { expect(results[0].error).toBeUndefined(); }); + it('errors when workspaceMode is static without workspace path', async () => { + const provider = new SequenceProvider('mock', { + responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }], + }); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases: [baseTestCase], + workspaceMode: 'static', + }), + ).rejects.toThrow('workspace.mode=static requires workspace.path or --workspace-path'); + }); + + it('errors when workspace path is combined with non-static workspaceMode', async () => { + const { mkdtemp } = await import('node:fs/promises'); + testDir = await mkdtemp(path.join(tmpdir(), 'agentv-ws-flag-')); + const provider = new SequenceProvider('mock', { + responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }], + }); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases: [baseTestCase], + workspacePath: testDir, + workspaceMode: 'temp', + }), + ).rejects.toThrow('--workspace-path requires --workspace-mode static when both are provided'); + }); + it('includes per-judge timing in scores', async () => { const provider = new SequenceProvider('mock', { responses: [ diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 6727c0e91..eacd573b2 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -211,11 +211,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/AZURE_OPENAI_API_KEY/i); }); - it('supports vscode configuration with optional workspace template from env var', () => { - const env = { - WORKSPACE_TEMPLATE_PATH: '/path/to/workspace.code-workspace', - } satisfies Record; - + it('supports vscode configuration with executable/wait/dry_run', () => { const target = resolveTargetDefinition( { name: 'editor', @@ -223,9 +219,8 @@ describe('resolveTargetDefinition', () => { executable: 'code-insiders', wait: false, dry_run: true, - workspace_template: '${{ WORKSPACE_TEMPLATE_PATH }}', }, - env, + {}, ); expect(target.kind).toBe('vscode'); @@ -236,7 +231,6 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('code-insiders'); expect(target.config.waitForResponse).toBe(false); expect(target.config.dryRun).toBe(true); - expect(target.config.workspaceTemplate).toBe('/path/to/workspace.code-workspace'); }); it('resolves vscode executable from env var', () => { @@ -488,159 +482,18 @@ describe('resolveTargetDefinition', () => { expect(target.config.args).toEqual(['--profile', 'default', '--model', 'gpt-4']); }); - it('resolves cli workspace_template with literal path', () => { - const target = resolveTargetDefinition( - { - name: 'cli-with-template', - provider: 'cli', - command: 'echo {PROMPT}', - workspace_template: '/templates/my-workspace', - }, - {}, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/my-workspace'); - expect(target.config.cwd).toBeUndefined(); - }); - - it('resolves cli workspace_template from environment variable', () => { - const env = { - WORKSPACE_DIR: '/path/to/workspace-template', - } satisfies Record; - - const target = resolveTargetDefinition( - { - name: 'cli-with-env-template', - provider: 'cli', - command: 'echo {PROMPT}', - workspace_template: '${{ WORKSPACE_DIR }}', - }, - env, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/path/to/workspace-template'); - }); - - it('throws when both cwd and workspace_template are specified for cli', () => { + it('rejects removed target-level workspace_template field', () => { expect(() => resolveTargetDefinition( { - name: 'cli-both', + name: 'cli-with-template', provider: 'cli', command: 'echo {PROMPT}', - cwd: '/some/path', workspace_template: '/templates/my-workspace', }, {}, ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves claude workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'claude-with-template', - provider: 'claude', - workspace_template: '/templates/claude-workspace', - }, - {}, - ); - - expect(target.kind).toBe('claude'); - if (target.kind !== 'claude') { - throw new Error('expected claude target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/claude-workspace'); - expect(target.config.cwd).toBeUndefined(); - }); - - it('throws when both cwd and workspace_template are specified for claude', () => { - expect(() => - resolveTargetDefinition( - { - name: 'claude-both', - provider: 'claude', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves codex workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'codex-with-template', - provider: 'codex', - workspace_template: '/templates/codex-workspace', - }, - {}, - ); - - expect(target.kind).toBe('codex'); - if (target.kind !== 'codex') { - throw new Error('expected codex target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/codex-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for codex', () => { - expect(() => - resolveTargetDefinition( - { - name: 'codex-both', - provider: 'codex', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves copilot-sdk workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'copilot-with-template', - provider: 'copilot-sdk', - workspace_template: '/templates/copilot-workspace', - }, - {}, - ); - - expect(target.kind).toBe('copilot-sdk'); - if (target.kind !== 'copilot-sdk') { - throw new Error('expected copilot-sdk target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/copilot-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for copilot-sdk', () => { - expect(() => - resolveTargetDefinition( - { - name: 'copilot-both', - provider: 'copilot-sdk', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); + ).toThrow(/workspace_template has been removed/i); }); it('resolves copilot alias to copilot-cli', () => { @@ -693,87 +546,18 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('copilot'); }); - it('resolves copilot-cli workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'copilot-cli-with-template', - provider: 'copilot-cli', - workspace_template: '/templates/copilot-cli-workspace', - }, - {}, - ); - - expect(target.kind).toBe('copilot-cli'); - if (target.kind !== 'copilot-cli') { - throw new Error('expected copilot-cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/copilot-cli-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for copilot-cli', () => { + it('rejects removed target-level workspaceTemplate camelCase field', () => { expect(() => resolveTargetDefinition( { - name: 'copilot-cli-both', - provider: 'copilot-cli', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves pi-coding-agent workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'pi-with-template', - provider: 'pi-coding-agent', - workspace_template: '/templates/pi-workspace', - }, - {}, - ); - - expect(target.kind).toBe('pi-coding-agent'); - if (target.kind !== 'pi-coding-agent') { - throw new Error('expected pi-coding-agent target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/pi-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for pi-coding-agent', () => { - expect(() => - resolveTargetDefinition( - { - name: 'pi-both', - provider: 'pi-coding-agent', - cwd: '/some/path', - workspace_template: '/templates/workspace', + name: 'cli-camel-case', + provider: 'cli', + command: 'echo {PROMPT}', + workspaceTemplate: '/templates/camel-case-workspace', }, {}, ), - ).toThrow(/mutually exclusive/i); - }); - - it('accepts workspaceTemplate camelCase variant', () => { - const target = resolveTargetDefinition( - { - name: 'cli-camel-case', - provider: 'cli', - command: 'echo {PROMPT}', - workspaceTemplate: '/templates/camel-case-workspace', - }, - {}, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/camel-case-workspace'); + ).toThrow(/workspace_template has been removed/i); }); }); diff --git a/packages/core/test/evaluation/repo-schema-validation.test.ts b/packages/core/test/evaluation/repo-schema-validation.test.ts index 1af9e5714..6b0a83766 100644 --- a/packages/core/test/evaluation/repo-schema-validation.test.ts +++ b/packages/core/test/evaluation/repo-schema-validation.test.ts @@ -91,6 +91,27 @@ describe('repo lifecycle schema validation', () => { expect(result.success).toBe(true); }); + it('accepts workspace.mode=temp', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'temp', + }, + }); + expect(result.success).toBe(true); + }); + + it('accepts workspace.path for static mode', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'static', + path: '/tmp/my-workspace', + }, + }); + expect(result.success).toBe(true); + }); + it('rejects invalid source type', () => { const result = EvalFileSchema.safeParse({ ...baseEval, @@ -148,6 +169,27 @@ describe('repo lifecycle schema validation', () => { expect(result.success).toBe(false); }); + it('rejects removed workspace.static_path field', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'static', + static_path: '/tmp/my-workspace', + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects removed workspace.pool field', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + pool: true, + }, + }); + expect(result.success).toBe(false); + }); + it('preserves existing workspace fields (template, hooks)', () => { const result = EvalFileSchema.safeParse({ ...baseEval, diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index c9bb56c64..e0e2352b5 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -277,6 +277,64 @@ tests: expect(cases[0].workspace?.isolation).toBe('per_test'); }); + it('infers workspace.mode=static when workspace.path is provided without mode', async () => { + const evalFile = path.join(testDir, 'workspace-path-implies-static.yaml'); + await writeFile( + evalFile, + ` +workspace: + path: /tmp/shared-workspace + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + const cases = await loadTests(evalFile, testDir); + expect(cases).toHaveLength(1); + expect(cases[0].workspace?.mode).toBe('static'); + expect(cases[0].workspace?.path).toBe('/tmp/shared-workspace'); + }); + + it('rejects removed workspace.static_path field', async () => { + const evalFile = path.join(testDir, 'workspace-static-path-removed.yaml'); + await writeFile( + evalFile, + ` +workspace: + mode: static + static_path: /tmp/shared-workspace + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + await expect(loadTests(evalFile, testDir)).rejects.toThrow(/workspace\.static_path has been removed/i); + }); + + it('rejects removed workspace.pool field', async () => { + const evalFile = path.join(testDir, 'workspace-pool-removed.yaml'); + await writeFile( + evalFile, + ` +workspace: + pool: true + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + await expect(loadTests(evalFile, testDir)).rejects.toThrow(/workspace\.pool has been removed/i); + }); + it('should handle case with no workspace config', async () => { const evalFile = path.join(testDir, 'no-workspace.yaml'); await writeFile( From a399a299cb8abbb62388990fde42e86f1a30afb0 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 01:46:09 +0000 Subject: [PATCH 2/8] feat(eval): canonicalize workspace mode/path configuration --- apps/cli/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/cli/README.md b/apps/cli/README.md index 61b827c3e..81ee7cc70 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -347,7 +347,6 @@ targets: - name: vscode_dev provider: vscode - workspace_template: ${{ WORKSPACE_PATH }} judge_target: azure-base - name: local_agent @@ -358,6 +357,8 @@ targets: Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`. +Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`). + Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields. ## Evaluation Features From 8eef8169eb73a0a55a229313e1ca6ad60195e38f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 01:47:11 +0000 Subject: [PATCH 3/8] style: apply biome formatting for workspace config changes --- packages/core/src/evaluation/orchestrator.ts | 5 +++-- packages/core/src/evaluation/yaml-parser.ts | 5 ++--- .../core/test/evaluation/workspace-config-parsing.test.ts | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 7cde86642..4b6c959ed 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -448,8 +448,9 @@ export async function runEvaluation( if (cliWorkspacePath && workspaceMode && workspaceMode !== 'static') { throw new Error('--workspace-path requires --workspace-mode static when both are provided'); } - const configuredMode = - cliWorkspacePath ? 'static' : (workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? 'static' : 'pooled')); + const configuredMode = cliWorkspacePath + ? 'static' + : (workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? 'static' : 'pooled')); const configuredStaticPath = cliWorkspacePath ?? yamlWorkspacePath; const useStaticWorkspace = configuredMode === 'static'; diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 5c9915654..6e388bb3d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -678,7 +678,7 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi const obj = raw as Record; if ('static_path' in obj) { throw new Error( - "workspace.static_path has been removed. Use workspace.path with workspace.mode=static.", + 'workspace.static_path has been removed. Use workspace.path with workspace.mode=static.', ); } if ('pool' in obj) { @@ -708,8 +708,7 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi const workspacePath = typeof obj.path === 'string' ? obj.path : undefined; const mode = explicitMode ?? (workspacePath ? 'static' : undefined); - if (!template && !isolation && !repos && !hooks && !mode && !workspacePath) - return undefined; + if (!template && !isolation && !repos && !hooks && !mode && !workspacePath) return undefined; return { ...(template !== undefined && { template }), diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index e0e2352b5..6f6960f73 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -314,7 +314,9 @@ tests: `, ); - await expect(loadTests(evalFile, testDir)).rejects.toThrow(/workspace\.static_path has been removed/i); + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /workspace\.static_path has been removed/i, + ); }); it('rejects removed workspace.pool field', async () => { From 174cdf8fdca42177a54b32230e4192c4d50e9c59 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 01:48:13 +0000 Subject: [PATCH 4/8] chore(schema): regenerate eval schema reference --- .../references/eval-schema.json | 2521 +++++++++++++---- 1 file changed, 2021 insertions(+), 500 deletions(-) diff --git a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json index 7779fb277..e5a8a742a 100644 --- a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json @@ -53,7 +53,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -67,20 +72,29 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file"] + "enum": [ + "text", + "file" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -115,7 +129,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -129,20 +148,29 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file"] + "enum": [ + "text", + "file" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -164,7 +192,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -178,20 +211,29 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file"] + "enum": [ + "text", + "file" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -228,7 +270,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -280,7 +325,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -310,7 +358,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -404,7 +455,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -420,7 +474,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -476,7 +532,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -492,7 +550,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -509,7 +570,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -526,13 +590,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -562,11 +631,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -607,7 +685,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -621,7 +704,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -632,7 +720,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -640,7 +730,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -654,7 +749,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -665,7 +765,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -695,7 +798,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -707,7 +813,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -729,17 +839,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -776,7 +895,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -813,7 +935,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -843,7 +968,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -858,7 +986,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -888,7 +1018,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -920,7 +1053,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -950,7 +1085,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -1004,7 +1142,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -1026,7 +1167,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1062,7 +1205,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -1098,7 +1244,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -1128,10 +1277,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1167,7 +1321,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -1248,7 +1405,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -1258,7 +1418,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -1295,7 +1458,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -1347,7 +1513,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -1377,7 +1546,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -1471,7 +1643,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -1487,7 +1662,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1543,7 +1720,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1559,7 +1738,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -1576,7 +1758,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -1593,13 +1778,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -1629,11 +1819,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -1674,7 +1873,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1688,7 +1892,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1699,7 +1908,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -1707,7 +1918,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1721,7 +1937,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -1732,7 +1953,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -1762,7 +1986,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -1774,7 +2001,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -1796,17 +2027,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -1843,7 +2083,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -1880,7 +2123,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -1910,7 +2156,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -1925,7 +2174,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -1955,7 +2206,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -1987,7 +2241,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2017,7 +2273,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -2071,7 +2330,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -2093,7 +2355,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2129,7 +2393,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -2165,7 +2432,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -2195,10 +2465,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2234,7 +2509,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -2315,7 +2593,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -2325,7 +2606,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -2374,7 +2658,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -2426,7 +2713,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -2456,7 +2746,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -2550,7 +2843,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -2566,7 +2862,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2622,7 +2920,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -2638,7 +2938,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -2655,7 +2958,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -2672,13 +2978,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -2708,11 +3019,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -2753,7 +3073,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2767,7 +3092,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2778,7 +3108,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -2786,7 +3118,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2800,7 +3137,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -2811,7 +3153,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -2841,7 +3186,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -2853,7 +3201,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -2875,17 +3227,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -2922,7 +3283,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -2959,7 +3323,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -2989,7 +3356,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -3004,7 +3374,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3034,7 +3406,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -3066,7 +3441,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3096,7 +3473,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -3150,7 +3530,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -3172,7 +3555,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3208,7 +3593,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3244,7 +3632,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3274,10 +3665,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3313,7 +3709,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -3394,7 +3793,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -3404,7 +3806,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -3441,7 +3846,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -3493,7 +3901,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -3523,7 +3934,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -3617,7 +4031,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -3633,7 +4050,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3689,7 +4108,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -3705,7 +4126,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -3722,7 +4146,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -3739,13 +4166,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -3775,11 +4207,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -3820,7 +4261,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3834,7 +4280,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3845,7 +4296,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -3853,7 +4306,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3867,7 +4325,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -3878,7 +4341,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -3908,7 +4374,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -3920,7 +4389,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -3942,17 +4415,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -3989,7 +4471,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -4026,7 +4511,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -4056,7 +4544,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -4071,7 +4562,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4101,7 +4594,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -4133,7 +4629,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4163,7 +4661,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -4217,7 +4718,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -4239,7 +4743,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4275,7 +4781,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4311,7 +4820,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4341,10 +4853,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -4380,7 +4897,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -4461,7 +4981,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -4471,7 +4994,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -4492,7 +5018,11 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -4503,7 +5033,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "total_budget_usd": { @@ -4531,7 +5063,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_test"] + "enum": [ + "shared", + "per_test" + ] }, "repos": { "type": "array", @@ -4555,7 +5090,10 @@ "format": "uri" } }, - "required": ["type", "url"], + "required": [ + "type", + "url" + ], "additionalProperties": false }, { @@ -4569,7 +5107,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false } ] @@ -4582,7 +5123,10 @@ }, "resolve": { "type": "string", - "enum": ["remote", "local"] + "enum": [ + "remote", + "local" + ] }, "ancestor": { "type": "integer", @@ -4611,7 +5155,10 @@ "additionalProperties": false } }, - "required": ["path", "source"], + "required": [ + "path", + "source" + ], "additionalProperties": false } }, @@ -4644,7 +5191,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -4675,7 +5226,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -4706,7 +5261,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -4737,7 +5296,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -4747,13 +5310,14 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": [ + "pooled", + "temp", + "static" + ] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false @@ -4772,7 +5336,9 @@ "type": "string" } }, - "required": ["id"], + "required": [ + "id" + ], "additionalProperties": false } }, @@ -4810,7 +5376,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -4824,20 +5395,29 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file"] + "enum": [ + "text", + "file" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -4859,7 +5439,12 @@ "properties": { "role": { "type": "string", - "enum": ["system", "user", "assistant", "tool"] + "enum": [ + "system", + "user", + "assistant", + "tool" + ] }, "content": { "anyOf": [ @@ -4873,20 +5458,29 @@ "properties": { "type": { "type": "string", - "enum": ["text", "file"] + "enum": [ + "text", + "file" + ] }, "value": { "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false } } ] } }, - "required": ["role", "content"], + "required": [ + "role", + "content" + ], "additionalProperties": false } } @@ -4923,7 +5517,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -4975,7 +5572,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -5005,7 +5605,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -5099,7 +5702,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -5115,7 +5721,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5171,7 +5779,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5187,7 +5797,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -5204,7 +5817,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -5221,13 +5837,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -5257,11 +5878,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -5302,7 +5932,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -5316,7 +5951,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -5327,7 +5967,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -5335,7 +5977,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -5349,7 +5996,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -5360,7 +6012,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -5390,7 +6045,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -5402,7 +6060,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -5424,17 +6086,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -5471,7 +6142,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -5508,7 +6182,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -5538,7 +6215,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -5553,7 +6233,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5583,7 +6265,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -5615,7 +6300,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5645,7 +6332,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -5699,7 +6389,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -5721,7 +6414,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5757,7 +6452,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5793,7 +6491,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5823,10 +6524,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -5862,7 +6568,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -5943,7 +6652,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -5953,7 +6665,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -5990,7 +6705,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -6042,7 +6760,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -6072,7 +6793,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -6166,7 +6890,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -6182,7 +6909,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6238,7 +6967,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6254,7 +6985,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -6271,7 +7005,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -6288,13 +7025,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -6324,11 +7066,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -6369,7 +7120,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6383,7 +7139,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6394,7 +7155,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -6402,7 +7165,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6416,7 +7184,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -6427,7 +7200,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -6457,7 +7233,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -6469,7 +7248,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -6491,17 +7274,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -6538,7 +7330,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -6575,7 +7370,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -6605,7 +7403,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -6620,7 +7421,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6650,7 +7453,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -6682,7 +7488,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6712,7 +7520,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -6766,7 +7577,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -6788,7 +7602,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6824,7 +7640,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -6860,7 +7679,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -6890,10 +7712,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -6929,7 +7756,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -7010,7 +7840,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -7020,7 +7853,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -7069,7 +7905,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -7121,7 +7960,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -7151,7 +7993,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -7245,7 +8090,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -7261,7 +8109,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7317,7 +8167,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7333,7 +8185,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -7350,7 +8205,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -7367,13 +8225,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -7403,11 +8266,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -7448,7 +8320,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -7462,7 +8339,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -7473,7 +8355,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -7481,7 +8365,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -7495,7 +8384,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -7506,7 +8400,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -7536,7 +8433,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -7548,7 +8448,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -7570,17 +8474,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -7617,7 +8530,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -7654,7 +8570,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -7684,7 +8603,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -7699,7 +8621,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7729,7 +8653,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -7761,7 +8688,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7791,7 +8720,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -7845,7 +8777,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -7867,7 +8802,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -7903,7 +8840,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -7939,7 +8879,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -7969,10 +8912,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8008,7 +8956,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -8089,7 +9040,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -8099,7 +9053,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -8136,7 +9093,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -8188,7 +9148,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -8218,7 +9181,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -8312,7 +9278,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -8328,7 +9297,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8384,7 +9355,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8400,7 +9373,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -8417,7 +9393,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -8434,13 +9413,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -8470,11 +9454,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -8515,7 +9508,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8529,7 +9527,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8540,7 +9543,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -8548,7 +9553,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8562,7 +9572,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -8573,7 +9588,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -8603,7 +9621,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -8615,7 +9636,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -8637,17 +9662,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -8684,7 +9718,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -8721,7 +9758,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -8751,7 +9791,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -8766,7 +9809,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8796,7 +9841,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -8828,7 +9876,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8858,7 +9908,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -8912,7 +9965,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -8934,7 +9990,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -8970,7 +10028,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9006,7 +10067,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9036,10 +10100,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9075,7 +10144,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -9156,7 +10228,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -9166,7 +10241,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -9187,7 +10265,11 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -9198,7 +10280,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "total_budget_usd": { @@ -9226,7 +10310,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_test"] + "enum": [ + "shared", + "per_test" + ] }, "repos": { "type": "array", @@ -9250,7 +10337,10 @@ "format": "uri" } }, - "required": ["type", "url"], + "required": [ + "type", + "url" + ], "additionalProperties": false }, { @@ -9264,7 +10354,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false } ] @@ -9277,7 +10370,10 @@ }, "resolve": { "type": "string", - "enum": ["remote", "local"] + "enum": [ + "remote", + "local" + ] }, "ancestor": { "type": "integer", @@ -9306,7 +10402,10 @@ "additionalProperties": false } }, - "required": ["path", "source"], + "required": [ + "path", + "source" + ], "additionalProperties": false } }, @@ -9339,7 +10438,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -9370,7 +10473,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -9401,7 +10508,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -9432,7 +10543,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -9442,13 +10557,14 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": [ + "pooled", + "temp", + "static" + ] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false @@ -9467,7 +10583,9 @@ "type": "string" } }, - "required": ["id"], + "required": [ + "id" + ], "additionalProperties": false } }, @@ -9522,7 +10640,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -9574,7 +10695,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -9604,7 +10728,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -9698,7 +10825,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -9714,7 +10844,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9770,7 +10902,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -9786,7 +10920,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -9803,7 +10940,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -9820,13 +10960,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -9856,11 +11001,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -9901,7 +11055,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9915,7 +11074,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9926,7 +11090,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -9934,7 +11100,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9948,7 +11119,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -9959,7 +11135,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -9989,7 +11168,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -10001,7 +11183,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -10023,17 +11209,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -10070,7 +11265,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -10107,7 +11305,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -10137,7 +11338,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -10152,7 +11356,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10182,7 +11388,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -10214,7 +11423,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10244,7 +11455,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -10298,7 +11512,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -10320,7 +11537,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10356,7 +11575,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10392,7 +11614,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10422,10 +11647,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10461,7 +11691,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -10542,7 +11775,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -10552,7 +11788,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -10589,7 +11828,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -10641,7 +11883,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -10671,7 +11916,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -10765,7 +12013,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -10781,7 +12032,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10837,7 +12090,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -10853,7 +12108,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -10870,7 +12128,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -10887,13 +12148,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -10923,11 +12189,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -10968,7 +12243,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10982,7 +12262,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -10993,7 +12278,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -11001,7 +12288,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11015,7 +12307,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -11026,7 +12323,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -11056,7 +12356,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -11068,7 +12371,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -11090,17 +12397,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -11137,7 +12453,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -11174,7 +12493,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -11204,7 +12526,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -11219,7 +12544,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11249,7 +12576,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -11281,7 +12611,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11311,7 +12643,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -11365,7 +12700,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -11387,7 +12725,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11423,7 +12763,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -11459,7 +12802,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -11489,10 +12835,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11528,7 +12879,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -11609,7 +12963,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -11619,7 +12976,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -11640,7 +13000,11 @@ }, "strategy": { "type": "string", - "enum": ["pass_at_k", "mean", "confidence_interval"] + "enum": [ + "pass_at_k", + "mean", + "confidence_interval" + ] }, "cost_limit_usd": { "type": "number", @@ -11651,7 +13015,9 @@ "minimum": 0 } }, - "required": ["count"], + "required": [ + "count" + ], "additionalProperties": false }, "total_budget_usd": { @@ -11702,7 +13068,10 @@ }, "type": { "type": "string", - "enum": ["code-judge", "code_judge"] + "enum": [ + "code-judge", + "code_judge" + ] }, "command": { "anyOf": [ @@ -11754,7 +13123,10 @@ "additionalProperties": {} } }, - "required": ["type", "command"], + "required": [ + "type", + "command" + ], "additionalProperties": false }, { @@ -11784,7 +13156,10 @@ }, "type": { "type": "string", - "enum": ["llm-judge", "llm_judge"] + "enum": [ + "llm-judge", + "llm_judge" + ] }, "prompt": { "anyOf": [ @@ -11878,7 +13253,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -11894,7 +13272,9 @@ "additionalProperties": {} } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11950,7 +13330,9 @@ } } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -11966,7 +13348,10 @@ "maximum": 1 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -11983,7 +13368,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false }, { @@ -12000,13 +13388,18 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false } ] } }, - "required": ["type", "aggregator"], + "required": [ + "type", + "aggregator" + ], "additionalProperties": false }, { @@ -12036,11 +13429,20 @@ }, "type": { "type": "string", - "enum": ["tool-trajectory", "tool_trajectory"] + "enum": [ + "tool-trajectory", + "tool_trajectory" + ] }, "mode": { "type": "string", - "enum": ["any_order", "in_order", "exact", "subset", "superset"] + "enum": [ + "any_order", + "in_order", + "exact", + "subset", + "superset" + ] }, "minimums": { "type": "object", @@ -12081,7 +13483,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -12095,7 +13502,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -12106,7 +13518,9 @@ ] } }, - "required": ["tool"], + "required": [ + "tool" + ], "additionalProperties": false } }, @@ -12114,7 +13528,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -12128,7 +13547,12 @@ "anyOf": [ { "type": "string", - "enum": ["exact", "ignore", "subset", "superset"] + "enum": [ + "exact", + "ignore", + "subset", + "superset" + ] }, { "type": "array", @@ -12139,7 +13563,10 @@ ] } }, - "required": ["type", "mode"], + "required": [ + "type", + "mode" + ], "additionalProperties": false }, { @@ -12169,7 +13596,10 @@ }, "type": { "type": "string", - "enum": ["field-accuracy", "field_accuracy"] + "enum": [ + "field-accuracy", + "field_accuracy" + ] }, "fields": { "type": "array", @@ -12181,7 +13611,11 @@ }, "match": { "type": "string", - "enum": ["exact", "numeric_tolerance", "date"] + "enum": [ + "exact", + "numeric_tolerance", + "date" + ] }, "required": { "type": "boolean" @@ -12203,17 +13637,26 @@ } } }, - "required": ["path", "match"], + "required": [ + "path", + "match" + ], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": ["weighted_average", "all_or_nothing"] + "enum": [ + "weighted_average", + "all_or_nothing" + ] } }, - "required": ["type", "fields"], + "required": [ + "type", + "fields" + ], "additionalProperties": false }, { @@ -12250,7 +13693,10 @@ "minimum": 0 } }, - "required": ["type", "threshold"], + "required": [ + "type", + "threshold" + ], "additionalProperties": false }, { @@ -12287,7 +13733,10 @@ "minimum": 0 } }, - "required": ["type", "budget"], + "required": [ + "type", + "budget" + ], "additionalProperties": false }, { @@ -12317,7 +13766,10 @@ }, "type": { "type": "string", - "enum": ["token-usage", "token_usage"] + "enum": [ + "token-usage", + "token_usage" + ] }, "max_total": { "type": "number", @@ -12332,7 +13784,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12362,7 +13816,10 @@ }, "type": { "type": "string", - "enum": ["execution-metrics", "execution_metrics"] + "enum": [ + "execution-metrics", + "execution_metrics" + ] }, "max_tool_calls": { "type": "number", @@ -12394,7 +13851,9 @@ "minimum": 0 } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12424,7 +13883,10 @@ }, "type": { "type": "string", - "enum": ["agent-judge", "agent_judge"] + "enum": [ + "agent-judge", + "agent_judge" + ] }, "prompt": { "type": "string" @@ -12478,7 +13940,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -12500,7 +13965,9 @@ "type": "string" } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12536,7 +14003,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -12572,7 +14042,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -12602,10 +14075,15 @@ }, "type": { "type": "string", - "enum": ["is-json", "is_json"] + "enum": [ + "is-json", + "is_json" + ] } }, - "required": ["type"], + "required": [ + "type" + ], "additionalProperties": false }, { @@ -12641,7 +14119,10 @@ "type": "string" } }, - "required": ["type", "value"], + "required": [ + "type", + "value" + ], "additionalProperties": false }, { @@ -12722,7 +14203,10 @@ "minLength": 1 } }, - "required": ["score_range", "outcome"], + "required": [ + "score_range", + "outcome" + ], "additionalProperties": false } } @@ -12732,7 +14216,10 @@ "minItems": 1 } }, - "required": ["type", "criteria"], + "required": [ + "type", + "criteria" + ], "additionalProperties": false } ] @@ -12748,7 +14235,10 @@ }, "isolation": { "type": "string", - "enum": ["shared", "per_test"] + "enum": [ + "shared", + "per_test" + ] }, "repos": { "type": "array", @@ -12772,7 +14262,10 @@ "format": "uri" } }, - "required": ["type", "url"], + "required": [ + "type", + "url" + ], "additionalProperties": false }, { @@ -12786,7 +14279,10 @@ "type": "string" } }, - "required": ["type", "path"], + "required": [ + "type", + "path" + ], "additionalProperties": false } ] @@ -12799,7 +14295,10 @@ }, "resolve": { "type": "string", - "enum": ["remote", "local"] + "enum": [ + "remote", + "local" + ] }, "ancestor": { "type": "integer", @@ -12828,7 +14327,10 @@ "additionalProperties": false } }, - "required": ["path", "source"], + "required": [ + "path", + "source" + ], "additionalProperties": false } }, @@ -12861,7 +14363,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12892,7 +14398,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12923,7 +14433,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12954,7 +14468,11 @@ }, "reset": { "type": "string", - "enum": ["none", "fast", "strict"] + "enum": [ + "none", + "fast", + "strict" + ] } }, "additionalProperties": false @@ -12964,13 +14482,14 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": [ + "pooled", + "temp", + "static" + ] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false @@ -12981,7 +14500,9 @@ ] } }, - "required": ["tests"], + "required": [ + "tests" + ], "additionalProperties": false } } From bb8086374f7c328b4d4a0b1bfdefd35723e944b5 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 01:49:03 +0000 Subject: [PATCH 5/8] style(schema): format generated eval schema --- .../references/eval-schema.json | 2506 ++++------------- 1 file changed, 488 insertions(+), 2018 deletions(-) diff --git a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json index e5a8a742a..d18defe6a 100644 --- a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json @@ -53,12 +53,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -72,29 +67,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file" - ] + "enum": ["text", "file"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -129,12 +115,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -148,29 +129,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file" - ] + "enum": ["text", "file"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -192,12 +164,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -211,29 +178,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file" - ] + "enum": ["text", "file"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -270,10 +228,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -325,10 +280,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -358,10 +310,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -455,10 +404,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -474,9 +420,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -532,9 +476,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -550,10 +492,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -570,10 +509,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -590,18 +526,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -631,20 +562,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -685,12 +607,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -704,12 +621,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -720,9 +632,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -730,12 +640,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -749,12 +654,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -765,10 +665,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -798,10 +695,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -813,11 +707,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -839,26 +729,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -895,10 +776,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -935,10 +813,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -968,10 +843,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -986,9 +858,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1018,10 +888,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -1053,9 +920,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1085,10 +950,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -1142,10 +1004,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -1167,9 +1026,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1205,10 +1062,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -1244,10 +1098,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -1277,15 +1128,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1321,10 +1167,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -1405,10 +1248,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -1418,10 +1258,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -1458,10 +1295,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -1513,10 +1347,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -1546,10 +1377,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -1643,10 +1471,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -1662,9 +1487,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1720,9 +1543,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -1738,10 +1559,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -1758,10 +1576,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -1778,18 +1593,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -1819,20 +1629,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -1873,12 +1674,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1892,12 +1688,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1908,9 +1699,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -1918,12 +1707,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1937,12 +1721,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -1953,10 +1732,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -1986,10 +1762,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -2001,11 +1774,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -2027,26 +1796,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -2083,10 +1843,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -2123,10 +1880,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -2156,10 +1910,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -2174,9 +1925,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2206,10 +1955,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -2241,9 +1987,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2273,10 +2017,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -2330,10 +2071,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -2355,9 +2093,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2393,10 +2129,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2432,10 +2165,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2465,15 +2195,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2509,10 +2234,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -2593,10 +2315,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -2606,10 +2325,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -2658,10 +2374,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -2713,10 +2426,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -2746,10 +2456,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -2843,10 +2550,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -2862,9 +2566,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2920,9 +2622,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -2938,10 +2638,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -2958,10 +2655,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -2978,18 +2672,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -3019,20 +2708,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -3073,12 +2753,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -3092,12 +2767,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -3108,9 +2778,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -3118,12 +2786,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -3137,12 +2800,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -3153,10 +2811,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -3186,10 +2841,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -3201,11 +2853,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -3227,26 +2875,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -3283,10 +2922,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -3323,10 +2959,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -3356,10 +2989,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -3374,9 +3004,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3406,10 +3034,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -3441,9 +3066,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3473,10 +3096,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -3530,10 +3150,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -3555,9 +3172,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3593,10 +3208,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3632,10 +3244,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3665,15 +3274,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -3709,10 +3313,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -3793,10 +3394,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -3806,10 +3404,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -3846,10 +3441,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -3901,10 +3493,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -3934,10 +3523,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -4031,10 +3617,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -4050,9 +3633,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4108,9 +3689,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4126,10 +3705,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -4146,10 +3722,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -4166,18 +3739,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -4207,20 +3775,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -4261,12 +3820,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4280,12 +3834,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4296,9 +3845,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -4306,12 +3853,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4325,12 +3867,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -4341,10 +3878,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -4374,10 +3908,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -4389,11 +3920,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -4415,26 +3942,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -4471,10 +3989,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -4511,10 +4026,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -4544,10 +4056,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -4562,9 +4071,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4594,10 +4101,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -4629,9 +4133,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4661,10 +4163,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -4718,10 +4217,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -4743,9 +4239,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4781,10 +4275,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4820,10 +4311,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4853,15 +4341,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -4897,10 +4380,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -4981,10 +4461,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -4994,10 +4471,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -5018,11 +4492,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -5033,9 +4503,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "total_budget_usd": { @@ -5063,10 +4531,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_test" - ] + "enum": ["shared", "per_test"] }, "repos": { "type": "array", @@ -5090,10 +4555,7 @@ "format": "uri" } }, - "required": [ - "type", - "url" - ], + "required": ["type", "url"], "additionalProperties": false }, { @@ -5107,10 +4569,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false } ] @@ -5123,10 +4582,7 @@ }, "resolve": { "type": "string", - "enum": [ - "remote", - "local" - ] + "enum": ["remote", "local"] }, "ancestor": { "type": "integer", @@ -5155,10 +4611,7 @@ "additionalProperties": false } }, - "required": [ - "path", - "source" - ], + "required": ["path", "source"], "additionalProperties": false } }, @@ -5191,11 +4644,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -5226,11 +4675,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -5261,11 +4706,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -5296,11 +4737,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -5310,11 +4747,7 @@ }, "mode": { "type": "string", - "enum": [ - "pooled", - "temp", - "static" - ] + "enum": ["pooled", "temp", "static"] }, "path": { "type": "string" @@ -5336,9 +4769,7 @@ "type": "string" } }, - "required": [ - "id" - ], + "required": ["id"], "additionalProperties": false } }, @@ -5376,12 +4807,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -5395,29 +4821,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file" - ] + "enum": ["text", "file"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -5439,12 +4856,7 @@ "properties": { "role": { "type": "string", - "enum": [ - "system", - "user", - "assistant", - "tool" - ] + "enum": ["system", "user", "assistant", "tool"] }, "content": { "anyOf": [ @@ -5458,29 +4870,20 @@ "properties": { "type": { "type": "string", - "enum": [ - "text", - "file" - ] + "enum": ["text", "file"] }, "value": { "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false } } ] } }, - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "additionalProperties": false } } @@ -5517,10 +4920,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -5572,10 +4972,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -5605,10 +5002,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -5702,10 +5096,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -5721,9 +5112,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5779,9 +5168,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -5797,10 +5184,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -5817,10 +5201,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -5837,18 +5218,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -5878,20 +5254,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -5932,12 +5299,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5951,12 +5313,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5967,9 +5324,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -5977,12 +5332,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -5996,12 +5346,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -6012,10 +5357,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -6045,10 +5387,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -6060,11 +5399,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -6086,26 +5421,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -6142,10 +5468,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -6182,10 +5505,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -6215,10 +5535,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -6233,9 +5550,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6265,10 +5580,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -6300,9 +5612,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6332,10 +5642,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -6389,10 +5696,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -6414,9 +5718,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6452,10 +5754,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -6491,10 +5790,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -6524,15 +5820,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6568,10 +5859,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -6652,10 +5940,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -6665,10 +5950,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -6705,10 +5987,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -6760,10 +6039,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -6793,10 +6069,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -6890,10 +6163,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -6909,9 +6179,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6967,9 +6235,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -6985,10 +6251,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -7005,10 +6268,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -7025,18 +6285,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -7066,20 +6321,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -7120,12 +6366,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7139,12 +6380,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7155,9 +6391,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -7165,12 +6399,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7184,12 +6413,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -7200,10 +6424,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -7233,10 +6454,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -7248,11 +6466,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -7274,26 +6488,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -7330,10 +6535,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -7370,10 +6572,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -7403,10 +6602,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -7421,9 +6617,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7453,10 +6647,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -7488,9 +6679,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7520,10 +6709,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -7577,10 +6763,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -7602,9 +6785,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7640,10 +6821,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7679,10 +6857,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7712,15 +6887,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -7756,10 +6926,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -7840,10 +7007,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -7853,10 +7017,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -7905,10 +7066,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -7960,10 +7118,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -7993,10 +7148,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -8090,10 +7242,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -8109,9 +7258,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8167,9 +7314,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8185,10 +7330,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -8205,10 +7347,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -8225,18 +7364,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -8266,20 +7400,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -8320,12 +7445,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -8339,12 +7459,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -8355,9 +7470,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -8365,12 +7478,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -8384,12 +7492,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -8400,10 +7503,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -8433,10 +7533,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -8448,11 +7545,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -8474,26 +7567,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -8530,10 +7614,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -8570,10 +7651,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -8603,10 +7681,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -8621,9 +7696,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8653,10 +7726,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -8688,9 +7758,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8720,10 +7788,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -8777,10 +7842,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -8802,9 +7864,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8840,10 +7900,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -8879,10 +7936,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -8912,15 +7966,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -8956,10 +8005,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -9040,10 +8086,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -9053,10 +8096,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -9093,10 +8133,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -9148,10 +8185,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -9181,10 +8215,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -9278,10 +8309,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -9297,9 +8325,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9355,9 +8381,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9373,10 +8397,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -9393,10 +8414,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -9413,18 +8431,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -9454,20 +8467,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -9508,12 +8512,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9527,12 +8526,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9543,9 +8537,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -9553,12 +8545,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9572,12 +8559,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -9588,10 +8570,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -9621,10 +8600,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -9636,11 +8612,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -9662,26 +8634,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -9718,10 +8681,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -9758,10 +8718,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -9791,10 +8748,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -9809,9 +8763,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9841,10 +8793,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -9876,9 +8825,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -9908,10 +8855,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -9965,10 +8909,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -9990,9 +8931,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10028,10 +8967,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10067,10 +9003,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10100,15 +9033,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10144,10 +9072,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -10228,10 +9153,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -10241,10 +9163,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -10265,11 +9184,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -10280,9 +9195,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "total_budget_usd": { @@ -10310,10 +9223,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_test" - ] + "enum": ["shared", "per_test"] }, "repos": { "type": "array", @@ -10337,10 +9247,7 @@ "format": "uri" } }, - "required": [ - "type", - "url" - ], + "required": ["type", "url"], "additionalProperties": false }, { @@ -10354,10 +9261,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false } ] @@ -10370,10 +9274,7 @@ }, "resolve": { "type": "string", - "enum": [ - "remote", - "local" - ] + "enum": ["remote", "local"] }, "ancestor": { "type": "integer", @@ -10402,10 +9303,7 @@ "additionalProperties": false } }, - "required": [ - "path", - "source" - ], + "required": ["path", "source"], "additionalProperties": false } }, @@ -10438,11 +9336,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -10473,11 +9367,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -10508,11 +9398,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -10543,11 +9429,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -10557,11 +9439,7 @@ }, "mode": { "type": "string", - "enum": [ - "pooled", - "temp", - "static" - ] + "enum": ["pooled", "temp", "static"] }, "path": { "type": "string" @@ -10583,9 +9461,7 @@ "type": "string" } }, - "required": [ - "id" - ], + "required": ["id"], "additionalProperties": false } }, @@ -10640,10 +9516,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -10695,10 +9568,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -10728,10 +9598,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -10825,10 +9692,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -10844,9 +9708,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10902,9 +9764,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -10920,10 +9780,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -10940,10 +9797,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -10960,18 +9814,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -11001,20 +9850,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -11055,12 +9895,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11074,12 +9909,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11090,9 +9920,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -11100,12 +9928,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11119,12 +9942,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -11135,10 +9953,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -11168,10 +9983,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -11183,11 +9995,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -11209,26 +10017,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -11265,10 +10064,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -11305,10 +10101,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -11338,10 +10131,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -11356,9 +10146,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11388,10 +10176,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -11423,9 +10208,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11455,10 +10238,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -11512,10 +10292,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -11537,9 +10314,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11575,10 +10350,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -11614,10 +10386,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -11647,15 +10416,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -11691,10 +10455,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -11775,10 +10536,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -11788,10 +10546,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -11828,10 +10583,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -11883,10 +10635,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -11916,10 +10665,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -12013,10 +10759,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -12032,9 +10775,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12090,9 +10831,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12108,10 +10847,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -12128,10 +10864,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -12148,18 +10881,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -12189,20 +10917,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -12243,12 +10962,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12262,12 +10976,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12278,9 +10987,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -12288,12 +10995,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12307,12 +11009,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -12323,10 +11020,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -12356,10 +11050,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -12371,11 +11062,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -12397,26 +11084,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -12453,10 +11131,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -12493,10 +11168,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -12526,10 +11198,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -12544,9 +11213,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12576,10 +11243,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -12611,9 +11275,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12643,10 +11305,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -12700,10 +11359,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -12725,9 +11381,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12763,10 +11417,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12802,10 +11453,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12835,15 +11483,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -12879,10 +11522,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -12963,10 +11603,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -12976,10 +11613,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -13000,11 +11634,7 @@ }, "strategy": { "type": "string", - "enum": [ - "pass_at_k", - "mean", - "confidence_interval" - ] + "enum": ["pass_at_k", "mean", "confidence_interval"] }, "cost_limit_usd": { "type": "number", @@ -13015,9 +11645,7 @@ "minimum": 0 } }, - "required": [ - "count" - ], + "required": ["count"], "additionalProperties": false }, "total_budget_usd": { @@ -13068,10 +11696,7 @@ }, "type": { "type": "string", - "enum": [ - "code-judge", - "code_judge" - ] + "enum": ["code-judge", "code_judge"] }, "command": { "anyOf": [ @@ -13123,10 +11748,7 @@ "additionalProperties": {} } }, - "required": [ - "type", - "command" - ], + "required": ["type", "command"], "additionalProperties": false }, { @@ -13156,10 +11778,7 @@ }, "type": { "type": "string", - "enum": [ - "llm-judge", - "llm_judge" - ] + "enum": ["llm-judge", "llm_judge"] }, "prompt": { "anyOf": [ @@ -13253,10 +11872,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -13272,9 +11888,7 @@ "additionalProperties": {} } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13330,9 +11944,7 @@ } } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13348,10 +11960,7 @@ "maximum": 1 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -13368,10 +11977,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false }, { @@ -13388,18 +11994,13 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false } ] } }, - "required": [ - "type", - "aggregator" - ], + "required": ["type", "aggregator"], "additionalProperties": false }, { @@ -13429,20 +12030,11 @@ }, "type": { "type": "string", - "enum": [ - "tool-trajectory", - "tool_trajectory" - ] + "enum": ["tool-trajectory", "tool_trajectory"] }, "mode": { "type": "string", - "enum": [ - "any_order", - "in_order", - "exact", - "subset", - "superset" - ] + "enum": ["any_order", "in_order", "exact", "subset", "superset"] }, "minimums": { "type": "object", @@ -13483,12 +12075,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -13502,12 +12089,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -13518,9 +12100,7 @@ ] } }, - "required": [ - "tool" - ], + "required": ["tool"], "additionalProperties": false } }, @@ -13528,12 +12108,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -13547,12 +12122,7 @@ "anyOf": [ { "type": "string", - "enum": [ - "exact", - "ignore", - "subset", - "superset" - ] + "enum": ["exact", "ignore", "subset", "superset"] }, { "type": "array", @@ -13563,10 +12133,7 @@ ] } }, - "required": [ - "type", - "mode" - ], + "required": ["type", "mode"], "additionalProperties": false }, { @@ -13596,10 +12163,7 @@ }, "type": { "type": "string", - "enum": [ - "field-accuracy", - "field_accuracy" - ] + "enum": ["field-accuracy", "field_accuracy"] }, "fields": { "type": "array", @@ -13611,11 +12175,7 @@ }, "match": { "type": "string", - "enum": [ - "exact", - "numeric_tolerance", - "date" - ] + "enum": ["exact", "numeric_tolerance", "date"] }, "required": { "type": "boolean" @@ -13637,26 +12197,17 @@ } } }, - "required": [ - "path", - "match" - ], + "required": ["path", "match"], "additionalProperties": false }, "minItems": 1 }, "aggregation": { "type": "string", - "enum": [ - "weighted_average", - "all_or_nothing" - ] + "enum": ["weighted_average", "all_or_nothing"] } }, - "required": [ - "type", - "fields" - ], + "required": ["type", "fields"], "additionalProperties": false }, { @@ -13693,10 +12244,7 @@ "minimum": 0 } }, - "required": [ - "type", - "threshold" - ], + "required": ["type", "threshold"], "additionalProperties": false }, { @@ -13733,10 +12281,7 @@ "minimum": 0 } }, - "required": [ - "type", - "budget" - ], + "required": ["type", "budget"], "additionalProperties": false }, { @@ -13766,10 +12311,7 @@ }, "type": { "type": "string", - "enum": [ - "token-usage", - "token_usage" - ] + "enum": ["token-usage", "token_usage"] }, "max_total": { "type": "number", @@ -13784,9 +12326,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13816,10 +12356,7 @@ }, "type": { "type": "string", - "enum": [ - "execution-metrics", - "execution_metrics" - ] + "enum": ["execution-metrics", "execution_metrics"] }, "max_tool_calls": { "type": "number", @@ -13851,9 +12388,7 @@ "minimum": 0 } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -13883,10 +12418,7 @@ }, "type": { "type": "string", - "enum": [ - "agent-judge", - "agent_judge" - ] + "enum": ["agent-judge", "agent_judge"] }, "prompt": { "type": "string" @@ -13940,10 +12472,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -13965,9 +12494,7 @@ "type": "string" } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -14003,10 +12530,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -14042,10 +12566,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -14075,15 +12596,10 @@ }, "type": { "type": "string", - "enum": [ - "is-json", - "is_json" - ] + "enum": ["is-json", "is_json"] } }, - "required": [ - "type" - ], + "required": ["type"], "additionalProperties": false }, { @@ -14119,10 +12635,7 @@ "type": "string" } }, - "required": [ - "type", - "value" - ], + "required": ["type", "value"], "additionalProperties": false }, { @@ -14203,10 +12716,7 @@ "minLength": 1 } }, - "required": [ - "score_range", - "outcome" - ], + "required": ["score_range", "outcome"], "additionalProperties": false } } @@ -14216,10 +12726,7 @@ "minItems": 1 } }, - "required": [ - "type", - "criteria" - ], + "required": ["type", "criteria"], "additionalProperties": false } ] @@ -14235,10 +12742,7 @@ }, "isolation": { "type": "string", - "enum": [ - "shared", - "per_test" - ] + "enum": ["shared", "per_test"] }, "repos": { "type": "array", @@ -14262,10 +12766,7 @@ "format": "uri" } }, - "required": [ - "type", - "url" - ], + "required": ["type", "url"], "additionalProperties": false }, { @@ -14279,10 +12780,7 @@ "type": "string" } }, - "required": [ - "type", - "path" - ], + "required": ["type", "path"], "additionalProperties": false } ] @@ -14295,10 +12793,7 @@ }, "resolve": { "type": "string", - "enum": [ - "remote", - "local" - ] + "enum": ["remote", "local"] }, "ancestor": { "type": "integer", @@ -14327,10 +12822,7 @@ "additionalProperties": false } }, - "required": [ - "path", - "source" - ], + "required": ["path", "source"], "additionalProperties": false } }, @@ -14363,11 +12855,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -14398,11 +12886,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -14433,11 +12917,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -14468,11 +12948,7 @@ }, "reset": { "type": "string", - "enum": [ - "none", - "fast", - "strict" - ] + "enum": ["none", "fast", "strict"] } }, "additionalProperties": false @@ -14482,11 +12958,7 @@ }, "mode": { "type": "string", - "enum": [ - "pooled", - "temp", - "static" - ] + "enum": ["pooled", "temp", "static"] }, "path": { "type": "string" @@ -14500,9 +12972,7 @@ ] } }, - "required": [ - "tests" - ], + "required": ["tests"], "additionalProperties": false } } From e8962c290d63de834a58581f8c500607fe63b3c1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 02:06:42 +0000 Subject: [PATCH 6/8] fix(example): use project-scoped marketplace in workspace setup script --- .../features/workspace-setup-script/README.md | 40 ++++++++++----- .../evals/dataset-vscode.eval.yaml | 6 +++ .../evals/dataset.eval.yaml | 4 ++ .../.claude-plugin/marketplace.json | 11 ++++ .../scripts/workspace-setup.mjs | 51 ++++++++++++++++++- .../.allagents/workspace.yaml | 8 +-- 6 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json diff --git a/examples/features/workspace-setup-script/README.md b/examples/features/workspace-setup-script/README.md index 71046d8b7..d2fdccfd4 100644 --- a/examples/features/workspace-setup-script/README.md +++ b/examples/features/workspace-setup-script/README.md @@ -1,6 +1,6 @@ # Workspace Setup Script -Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, including sourcing plugin files (like `AGENTS.md` and prompt files) into the workspace. +Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, then register a project-scoped marketplace and sync plugin content (including prompt files). ## Problem @@ -8,18 +8,21 @@ Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an a ## Solution -A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes the stale `.allagents/` directory, and runs `allagents workspace init` with the template path passed via `--from`. +A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes stale `.allagents/` state, runs `allagents workspace init --from`, registers a project-scoped marketplace, then runs `allagents workspace sync`. ``` workspace-setup-script/ ├── evals/ │ └── dataset.eval.yaml # Eval with before_all hook ├── plugins/ -│ └── my-plugin/ # External plugin (sourced by allagents) +│ └── my-plugin/ # Plugin content (AGENTS + prompt) │ ├── AGENTS.md # Agent guidelines │ └── .github/ │ └── prompts/ │ └── summarize-repo.prompt.md +├── marketplace/ +│ └── .claude-plugin/ +│ └── marketplace.json # Local marketplace manifest ├── scripts/ │ └── workspace-setup.mjs # Generic setup script (reusable across evals) └── workspace-template/ @@ -27,23 +30,27 @@ workspace-setup-script/ └── workspace.yaml # Template for allagents init ``` -## Sourcing plugin files with allagents +## Plugin Installation via Project Marketplace -The `plugins/` directory lives outside `workspace-template/` as an external source. The `.allagents/workspace.yaml` uses `workspace.source` and `workspace.files` to tell allagents to copy specific files to the workspace root: +The `.allagents/workspace.yaml` installs a plugin from a named marketplace: ```yaml # .allagents/workspace.yaml -workspace: - source: ../plugins/my-plugin/ - files: - - AGENTS.md +plugins: + - my-plugin@workspace-setup-script-marketplace ``` -The source path is relative to the workspace template root (parent of `.allagents/`). When `npx allagents workspace init --from` runs, it resolves `../plugins/my-plugin/` from the template location and copies `AGENTS.md` to the workspace root. +The setup script registers that marketplace using project scope: + +```bash +npx --yes allagents plugin marketplace add ../marketplace --scope project +``` + +This matches the project-scoped marketplace flow introduced in `allagents` (PR #224). ## Eval YAML -The template path is passed as an argument. Use `--require` to validate that expected artifacts exist in the workspace after initialization: +The template path and local marketplace path are passed as arguments. Use `--require` to validate expected artifacts after sync: ```yaml workspace: @@ -55,8 +62,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace - --require - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md ``` The `--require` flag accepts one or more file paths (relative to the workspace root). If any required file is missing after `allagents workspace init`, the script exits with an error listing the missing files. @@ -83,9 +94,10 @@ The `type: file` path is resolved from the eval file's directory up to the repo 1. AgentV copies `workspace-template/` to a pooled workspace 2. The setup script removes stale `.allagents/` config and runs `npx allagents workspace init` -3. Allagents reads `workspace.source`/`workspace.files` and copies `AGENTS.md` from the plugin to the workspace root -4. The `--require AGENTS.md` check validates the artifact exists -5. AgentV clones repos and runs tests against the initialized workspace +3. The setup script registers the local marketplace with `--scope project` +4. `allagents workspace sync` installs `my-plugin@workspace-setup-script-marketplace` +5. `--require` checks verify `AGENTS.md` and `.github/prompts/summarize-repo.prompt.md` exist +6. AgentV clones repos and runs tests against the initialized workspace ## Cross-platform diff --git a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml index d4f1c43f7..4e6872983 100644 --- a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml @@ -11,6 +11,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace + - --require + - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md after_each: reset: fast repos: diff --git a/examples/features/workspace-setup-script/evals/dataset.eval.yaml b/examples/features/workspace-setup-script/evals/dataset.eval.yaml index 7d2b81d80..406b6372a 100644 --- a/examples/features/workspace-setup-script/evals/dataset.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset.eval.yaml @@ -11,8 +11,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace - --require - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md repos: - path: ./my-repo source: diff --git a/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json b/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json new file mode 100644 index 000000000..d7cef0488 --- /dev/null +++ b/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json @@ -0,0 +1,11 @@ +{ + "name": "workspace-setup-script-marketplace", + "description": "Local marketplace used by the workspace setup script example", + "plugins": [ + { + "name": "my-plugin", + "description": "Demo plugin containing AGENTS.md and summarize prompt", + "source": "../plugins/my-plugin" + } + ] +} diff --git a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs index 3f825f558..7c45ee286 100644 --- a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs +++ b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs @@ -22,13 +22,13 @@ import { spawnSync } from 'node:child_process'; import { cpSync, existsSync, readFileSync, rmSync } from 'node:fs'; -import { basename, join } from 'node:path'; +import { basename, isAbsolute, join, resolve } from 'node:path'; // --- parse arguments --- const fromIndex = process.argv.indexOf('--from'); if (fromIndex === -1 || !process.argv[fromIndex + 1]) { console.error( - 'Usage: workspace-setup.mjs --from [--source ...] [--require ...]', + 'Usage: workspace-setup.mjs --from [--source ...] [--marketplace-source ] [--marketplace-name ] [--require ...]', ); process.exit(1); } @@ -52,6 +52,14 @@ for (let i = 0; i < process.argv.length; i++) { } } +// Optional project-scoped marketplace source to register after init. +const marketplaceSourceIndex = process.argv.indexOf('--marketplace-source'); +const marketplaceSource = + marketplaceSourceIndex !== -1 ? process.argv[marketplaceSourceIndex + 1] : undefined; +const marketplaceNameIndex = process.argv.indexOf('--marketplace-name'); +const marketplaceName = + marketplaceNameIndex !== -1 ? process.argv[marketplaceNameIndex + 1] : undefined; + // --- stdin context from AgentV --- const { workspace_path } = JSON.parse(readFileSync(0, 'utf8')); if (!workspace_path) { @@ -89,6 +97,45 @@ if (result.status !== 0) { process.exit(result.status ?? 1); } +// --- optionally register project-scoped marketplace and resync --- +if (marketplaceSource) { + const resolvedMarketplaceSource = isAbsolute(marketplaceSource) + ? marketplaceSource + : resolve(process.cwd(), marketplaceSource); + + const addMarketplaceArgs = [ + '--yes', + 'allagents', + 'plugin', + 'marketplace', + 'add', + resolvedMarketplaceSource, + '--scope', + 'project', + ]; + if (marketplaceName) { + addMarketplaceArgs.push('--name', marketplaceName); + } + + const addMarketplaceResult = spawnSync(npx, addMarketplaceArgs, { + stdio: ['ignore', 'inherit', 'inherit'], + shell: process.platform === 'win32', + cwd: workspace_path, + }); + if (addMarketplaceResult.status !== 0) { + process.exit(addMarketplaceResult.status ?? 1); + } + + const syncResult = spawnSync(npx, ['--yes', 'allagents', 'workspace', 'sync'], { + stdio: ['ignore', 'inherit', 'inherit'], + shell: process.platform === 'win32', + cwd: workspace_path, + }); + if (syncResult.status !== 0) { + process.exit(syncResult.status ?? 1); + } +} + // --- validate required artifacts exist in workspace --- const missing = requiredFiles.filter((file) => !existsSync(join(workspace_path, file))); if (missing.length > 0) { diff --git a/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml b/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml index a27617f47..9da75a4fe 100644 --- a/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml +++ b/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml @@ -4,12 +4,8 @@ repositories: repo: agentv description: Example repository for workspace setup demo -plugins: [] +plugins: + - my-plugin@workspace-setup-script-marketplace clients: - copilot - -workspace: - source: ../plugins/my-plugin/ - files: - - AGENTS.md From dda8e29ed9867905ca4516f1e4523b05a4417ad6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 02:53:32 +0000 Subject: [PATCH 7/8] fix(validate): reject removed target workspace_template fields --- .../validation/targets-validator.ts | 11 +++ .../validation/targets-validator.test.ts | 67 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 packages/core/test/evaluation/validation/targets-validator.test.ts diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index ef5fee2c9..ac064f4c8 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -240,6 +240,7 @@ function validateUnknownSettings( location: string, errors: ValidationError[], ): void { + const removedTargetFields = new Set(['workspace_template', 'workspaceTemplate']); const knownSettings = getKnownSettings(provider); if (!knownSettings) { // Unknown provider, skip settings validation @@ -250,6 +251,16 @@ function validateUnknownSettings( const baseFields = new Set(['name', 'provider', 'judge_target', 'workers', '$schema', 'targets']); for (const key of Object.keys(target)) { + if (removedTargetFields.has(key)) { + errors.push({ + severity: 'error', + filePath: absolutePath, + location: `${location}.${key}`, + message: 'target-level workspace_template has been removed. Use eval-level workspace.template.', + }); + continue; + } + if (!baseFields.has(key) && !knownSettings.has(key)) { errors.push({ severity: 'warning', diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts new file mode 100644 index 000000000..f42ae8078 --- /dev/null +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -0,0 +1,67 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { validateTargetsFile } from '../../../src/evaluation/validation/targets-validator.js'; + +describe('validateTargetsFile', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-targets-validator-test-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('rejects removed target-level workspace_template field', async () => { + const filePath = path.join(tempDir, 'removed-workspace-template.yaml'); + await writeFile( + filePath, + `targets: + - name: test-target + provider: codex-cli + workspace_template: ./template +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].workspace_template' && + error.message.includes('workspace_template has been removed'), + ), + ).toBe(true); + }); + + it('rejects removed target-level workspaceTemplate camelCase field', async () => { + const filePath = path.join(tempDir, 'removed-workspace-template-camel.yaml'); + await writeFile( + filePath, + `targets: + - name: test-target + provider: codex-cli + workspaceTemplate: ./template +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].workspaceTemplate' && + error.message.includes('workspace_template has been removed'), + ), + ).toBe(true); + }); +}); From 85960a030bca17ada6b0867bada551be3d2cdfd8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 10 Mar 2026 02:54:21 +0000 Subject: [PATCH 8/8] style(validate): satisfy biome formatting --- packages/core/src/evaluation/validation/targets-validator.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index ac064f4c8..c507308e9 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -256,7 +256,8 @@ function validateUnknownSettings( severity: 'error', filePath: absolutePath, location: `${location}.${key}`, - message: 'target-level workspace_template has been removed. Use eval-level workspace.template.', + message: + 'target-level workspace_template has been removed. Use eval-level workspace.template.', }); continue; }