feat: canonical workspace.mode + workspace.path model

README.md

@@ -347,7 +347,6 @@ targets:
 
   - name: vscode_dev
     provider: vscode
-    workspace_template: ${{ WORKSPACE_PATH }}
     judge_target: azure-base
 
   - name: local_agent
@@ -358,6 +357,8 @@ targets:
 
 Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`.
 
+Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`).
+
 Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields.
 
 ## Evaluation Features

apps/cli/README.md

@@ -347,7 +347,6 @@ targets:
 
   - name: vscode_dev
     provider: vscode
-    workspace_template: ${{ WORKSPACE_PATH }}
     judge_target: azure-base
 
   - name: local_agent
@@ -358,6 +357,8 @@ targets:
 
 Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`.
 
+Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`).
+
 Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields.
 
 ## Evaluation Features

apps/cli/src/commands/eval/commands/run.ts

@@ -105,53 +105,16 @@ export const evalRunCommand = command({
       long: 'verbose',
       description: 'Enable verbose logging',
     }),
-    keepWorkspaces: flag({
-      long: 'keep-workspaces',
-      description:
-        'Always keep temporary workspaces after evaluation (default: keep on failure only)',
-    }),
-    cleanupWorkspaces: flag({
-      long: 'cleanup-workspaces',
-      description: 'Always cleanup temporary workspaces, even on failure',
-    }),
-    poolWorkspaces: flag({
-      long: 'pool-workspaces',
-      description: 'Enable workspace pooling (default for shared workspaces with repos)',
-    }),
-    noPool: flag({
-      long: 'no-pool',
-      description: 'Disable workspace pooling (clone fresh each run)',
-    }),
-    workspace: option({
-      type: optional(string),
-      long: 'workspace',
-      description: 'Use an existing directory as the workspace directly (skips clone/copy/pool)',
-    }),
     workspaceMode: option({
       type: optional(string),
       long: 'workspace-mode',
-      description: "Workspace mode: 'pooled', 'ephemeral', or 'static'",
+      description: "Workspace mode: 'pooled', 'temp', or 'static'",
     }),
     workspacePath: option({
       type: optional(string),
       long: 'workspace-path',
       description: 'Static workspace directory path (used when workspace mode is static)',
     }),
-    workspaceClean: option({
-      type: optional(string),
-      long: 'workspace-clean',
-      description: "Pooled reset clean mode: 'standard' or 'full'",
-    }),
-    retainOnSuccess: option({
-      type: optional(string),
-      long: 'retain-on-success',
-      description: "Workspace retention on success: 'keep' or 'cleanup'",
-    }),
-    retainOnFailure: option({
-      type: optional(string),
-      long: 'retain-on-failure',
-      description: "Workspace retention on failure: 'keep' or 'cleanup'",
-    }),
     otelFile: option({
       type: optional(string),
       long: 'otel-file',
@@ -216,16 +179,8 @@ export const evalRunCommand = command({
       cache: args.cache,
       noCache: args.noCache,
       verbose: args.verbose,
-      keepWorkspaces: args.keepWorkspaces,
-      cleanupWorkspaces: args.cleanupWorkspaces,
-      poolWorkspaces: args.poolWorkspaces,
-      noPool: args.noPool,
-      workspace: args.workspace,
       workspaceMode: args.workspaceMode,
       workspacePath: args.workspacePath,
-      workspaceClean: args.workspaceClean,
-      retainOnSuccess: args.retainOnSuccess,
-      retainOnFailure: args.retainOnFailure,
       trace: false,
       otelFile: args.otelFile,
       traceFile: args.traceFile,

apps/cli/src/commands/eval/run-eval.ts

@@ -69,23 +69,15 @@ interface NormalizedOptions {
   readonly cache: boolean;
   readonly noCache: boolean;
   readonly verbose: boolean;
-  readonly keepWorkspaces: boolean;
-  readonly cleanupWorkspaces: boolean;
   readonly otelFile?: string;
   readonly traceFile?: string;
   readonly exportOtel: boolean;
   readonly otelBackend?: string;
   readonly otelCaptureContent: boolean;
   readonly otelGroupTurns: boolean;
   readonly retryErrors?: string;
-  readonly poolWorkspaces?: boolean;
-  readonly poolMaxSlots?: number;
-  readonly workspace?: string;
-  readonly workspaceMode?: 'pooled' | 'ephemeral' | 'static';
+  readonly workspaceMode?: 'pooled' | 'temp' | 'static';
   readonly workspacePath?: string;
-  readonly workspaceClean?: 'standard' | 'full';
-  readonly retainOnSuccess?: 'keep' | 'cleanup';
-  readonly retainOnFailure?: 'keep' | 'cleanup';
 }
 
 function normalizeBoolean(value: unknown): boolean {
@@ -134,16 +126,8 @@ function normalizeOptionalNumber(value: unknown): number | undefined {
   return undefined;
 }
 
-function normalizeWorkspaceMode(value: unknown): 'pooled' | 'ephemeral' | 'static' | undefined {
-  return value === 'pooled' || value === 'ephemeral' || value === 'static' ? value : undefined;
-}
-
-function normalizeWorkspaceClean(value: unknown): 'standard' | 'full' | undefined {
-  return value === 'standard' || value === 'full' ? value : undefined;
-}
-
-function normalizeRetention(value: unknown): 'keep' | 'cleanup' | undefined {
-  return value === 'keep' || value === 'cleanup' ? value : undefined;
+function normalizeWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined {
+  return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined;
 }
 
 function normalizeOptions(
@@ -198,6 +182,18 @@ function normalizeOptions(
   // Output dir: CLI --out > config output.dir > auto-generated
   const cliOut = normalizeString(rawOptions.out);
   const configOut = config?.output?.dir;
+  const cliWorkspacePath = normalizeString(rawOptions.workspacePath);
+  const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode);
+  const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode);
+  if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== 'static') {
+    throw new Error('--workspace-path requires --workspace-mode=static (or omit --workspace-mode)');
+  }
+
+  const yamlExecutionRecord = yamlExecution as Record<string, unknown> | undefined;
+  const yamlWorkspaceMode = normalizeWorkspaceMode(yamlExecutionRecord?.workspace_mode);
+  const yamlWorkspacePath = normalizeString(yamlExecutionRecord?.workspace_path);
+  const workspacePath = cliWorkspacePath ?? yamlWorkspacePath;
+  const workspaceMode = cliWorkspacePath ? 'static' : (cliWorkspaceMode ?? yamlWorkspaceMode);
 
   return {
     target: singleTarget,
@@ -223,11 +219,6 @@ function normalizeOptions(
       normalizeBoolean(rawOptions.verbose) ||
       yamlExecution?.verbose === true ||
       config?.execution?.verbose === true,
-    keepWorkspaces:
-      normalizeBoolean(rawOptions.keepWorkspaces) ||
-      yamlExecution?.keep_workspaces === true ||
-      config?.execution?.keepWorkspaces === true,
-    cleanupWorkspaces: normalizeBoolean(rawOptions.cleanupWorkspaces),
     // Precedence: CLI > YAML config > TS config
     otelFile:
       normalizeString(rawOptions.otelFile) ??
@@ -250,23 +241,8 @@ function normalizeOptions(
     otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent),
     otelGroupTurns: normalizeBoolean(rawOptions.otelGroupTurns),
     retryErrors: normalizeString(rawOptions.retryErrors),
-    // Pool: --no-pool explicitly disables; --pool-workspaces explicitly enables; YAML config; default undefined (orchestrator defaults to true)
-    poolWorkspaces: normalizeBoolean(rawOptions.noPool)
-      ? false
-      : normalizeBoolean(rawOptions.poolWorkspaces)
-        ? true
-        : yamlExecution?.pool_workspaces,
-    poolMaxSlots: yamlExecution?.pool_slots,
-    workspace: normalizeString(rawOptions.workspace),
-    workspaceMode: normalizeWorkspaceMode(rawOptions.workspaceMode),
-    workspacePath: normalizeString(rawOptions.workspacePath),
-    workspaceClean: normalizeWorkspaceClean(rawOptions.workspaceClean),
-    retainOnSuccess:
-      normalizeRetention(rawOptions.retainOnSuccess) ??
-      (normalizeBoolean(rawOptions.keepWorkspaces) ? 'keep' : undefined),
-    retainOnFailure:
-      normalizeRetention(rawOptions.retainOnFailure) ??
-      (normalizeBoolean(rawOptions.cleanupWorkspaces) ? 'cleanup' : undefined),
+    workspaceMode,
+    workspacePath,
   } satisfies NormalizedOptions;
 }
 
@@ -578,7 +554,6 @@ async function runSingleEvalFile(params: {
 
   // Create streaming observer for real-time OTel span export
   const streamingObserver = otelExporter?.createStreamingObserver() ?? null;
-
   const results = await evaluationRunner({
     testFilePath,
     repoRoot,
@@ -604,16 +579,8 @@ async function runSingleEvalFile(params: {
     evalCases,
     verbose: options.verbose,
     maxConcurrency: resolvedWorkers,
-    keepWorkspaces: options.keepWorkspaces,
-    cleanupWorkspaces: options.cleanupWorkspaces,
-    poolWorkspaces: options.poolWorkspaces,
-    poolMaxSlots: options.poolMaxSlots,
-    workspace: options.workspace,
     workspaceMode: options.workspaceMode,
     workspacePath: options.workspacePath,
-    workspaceClean: options.workspaceClean,
-    retainOnSuccess: options.retainOnSuccess,
-    retainOnFailure: options.retainOnFailure,
     trials: trialsConfig,
     totalBudgetUsd,
     failOnError,
@@ -717,30 +684,22 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise<void>
     retryNonErrorResults = await loadNonErrorResults(retryPath);
   }
 
-  if (options.keepWorkspaces && options.cleanupWorkspaces) {
-    console.warn(
-      'Warning: Both --keep-workspaces and --cleanup-workspaces specified. --cleanup-workspaces takes precedence.',
-    );
-  }
-
   // Validate static workspace path exists and is a directory
-  const explicitWorkspacePath = options.workspacePath ?? options.workspace;
-  if (explicitWorkspacePath) {
-    const resolvedWorkspace = path.resolve(explicitWorkspacePath);
+  if (options.workspacePath) {
+    const resolvedWorkspace = path.resolve(options.workspacePath);
     try {
       const { stat } = await import('node:fs/promises');
       const stats = await stat(resolvedWorkspace);
       if (!stats.isDirectory()) {
-        throw new Error(`--workspace path is not a directory: ${resolvedWorkspace}`);
+        throw new Error(`--workspace-path is not a directory: ${resolvedWorkspace}`);
       }
     } catch (err) {
       if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
-        throw new Error(`--workspace path does not exist: ${resolvedWorkspace}`);
+        throw new Error(`--workspace-path does not exist: ${resolvedWorkspace}`);
       }
       throw err;
     }
-    // Update workspace paths to resolved absolute path
-    options = { ...options, workspace: resolvedWorkspace, workspacePath: resolvedWorkspace };
+    options = { ...options, workspacePath: resolvedWorkspace };
   }
 
   if (options.verbose) {

examples/features/workspace-setup-script/README.md

@@ -1,49 +1,56 @@
 # Workspace Setup Script
 
-Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, including sourcing plugin files (like `AGENTS.md` and prompt files) into the workspace.
+Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, then register a project-scoped marketplace and sync plugin content (including prompt files).
 
 ## Problem
 
 `allagents workspace init` fails if `.allagents/workspace.yaml` already exists. In CI and repeated eval runs, stale artifacts need to be cleaned first. Without a wrapper, you'd need shell operators like `&&` (not cross-platform) or framework-level multi-command support.
 
 ## Solution
 
-A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes the stale `.allagents/` directory, and runs `allagents workspace init` with the template path passed via `--from`.
+A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes stale `.allagents/` state, runs `allagents workspace init --from`, registers a project-scoped marketplace, then runs `allagents workspace sync`.
 
 ```
 workspace-setup-script/
 ├── evals/
 │   └── dataset.eval.yaml        # Eval with before_all hook
 ├── plugins/
-│   └── my-plugin/               # External plugin (sourced by allagents)
+│   └── my-plugin/               # Plugin content (AGENTS + prompt)
 │       ├── AGENTS.md             # Agent guidelines
 │       └── .github/
 │           └── prompts/
 │               └── summarize-repo.prompt.md
+├── marketplace/
+│   └── .claude-plugin/
+│       └── marketplace.json     # Local marketplace manifest
 ├── scripts/
 │   └── workspace-setup.mjs      # Generic setup script (reusable across evals)
 └── workspace-template/
     └── .allagents/
         └── workspace.yaml       # Template for allagents init
 ```
 
-## Sourcing plugin files with allagents
+## Plugin Installation via Project Marketplace
 
-The `plugins/` directory lives outside `workspace-template/` as an external source. The `.allagents/workspace.yaml` uses `workspace.source` and `workspace.files` to tell allagents to copy specific files to the workspace root:
+The `.allagents/workspace.yaml` installs a plugin from a named marketplace:
 
 ```yaml
 # .allagents/workspace.yaml
-workspace:
-  source: ../plugins/my-plugin/
-  files:
-    - AGENTS.md
+plugins:
+  - my-plugin@workspace-setup-script-marketplace
 ```
 
-The source path is relative to the workspace template root (parent of `.allagents/`). When `npx allagents workspace init --from` runs, it resolves `../plugins/my-plugin/` from the template location and copies `AGENTS.md` to the workspace root.
+The setup script registers that marketplace using project scope:
+
+```bash
+npx --yes allagents plugin marketplace add ../marketplace --scope project
+```
+
+This matches the project-scoped marketplace flow introduced in `allagents` (PR #224).
 
 ## Eval YAML
 
-The template path is passed as an argument. Use `--require` to validate that expected artifacts exist in the workspace after initialization:
+The template path and local marketplace path are passed as arguments. Use `--require` to validate expected artifacts after sync:
 
 ```yaml
 workspace:
@@ -55,8 +62,12 @@ workspace:
         - ../scripts/workspace-setup.mjs
         - --from
         - ../workspace-template/.allagents/workspace.yaml
+        - --marketplace-source
+        - ../marketplace
         - --require
         - AGENTS.md
+        - --require
+        - .github/prompts/summarize-repo.prompt.md
 ```
 
 The `--require` flag accepts one or more file paths (relative to the workspace root). If any required file is missing after `allagents workspace init`, the script exits with an error listing the missing files.
@@ -83,9 +94,10 @@ The `type: file` path is resolved from the eval file's directory up to the repo
 
 1. AgentV copies `workspace-template/` to a pooled workspace
 2. The setup script removes stale `.allagents/` config and runs `npx allagents workspace init`
-3. Allagents reads `workspace.source`/`workspace.files` and copies `AGENTS.md` from the plugin to the workspace root
-4. The `--require AGENTS.md` check validates the artifact exists
-5. AgentV clones repos and runs tests against the initialized workspace
+3. The setup script registers the local marketplace with `--scope project`
+4. `allagents workspace sync` installs `my-plugin@workspace-setup-script-marketplace`
+5. `--require` checks verify `AGENTS.md` and `.github/prompts/summarize-repo.prompt.md` exist
+6. AgentV clones repos and runs tests against the initialized workspace
 
 ## Cross-platform
 

examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml

@@ -11,6 +11,12 @@ workspace:
         - ../scripts/workspace-setup.mjs
         - --from
         - ../workspace-template/.allagents/workspace.yaml
+        - --marketplace-source
+        - ../marketplace
+        - --require
+        - AGENTS.md
+        - --require
+        - .github/prompts/summarize-repo.prompt.md
     after_each:
       reset: fast
   repos:

examples/features/workspace-setup-script/evals/dataset.eval.yaml

@@ -11,8 +11,12 @@ workspace:
         - ../scripts/workspace-setup.mjs
         - --from
         - ../workspace-template/.allagents/workspace.yaml
+        - --marketplace-source
+        - ../marketplace
         - --require
         - AGENTS.md
+        - --require
+        - .github/prompts/summarize-repo.prompt.md
   repos:
     - path: ./my-repo
       source: