diff --git a/README.md b/README.md index 4f83e9b..6dbc420 100644 --- a/README.md +++ b/README.md @@ -249,8 +249,12 @@ answerlayer optimize start \ --mode recommendation ``` -By default the proposal model also reviews bounded failed-case traces during -failure analysis. Use `--analysis-model MODEL` to select a dedicated reviewer. +The shared `--model` defaults the eval agent, judge, trace analyzer, and +component author. Use `--eval-model`, `--judge-model`, `--analysis-model`, or +`--proposal-model` for explicit role overrides. If `--analysis-model` is +omitted, trace analysis uses the explicit `--proposal-model` before falling +back to the shared model. Without a shared model or `--judge-model`, each case +keeps its configured judge. Inspect and operate the registry with `answerlayer optimize list`, `get RUN_ID`, `pause RUN_ID`, `resume RUN_ID`, `stop RUN_ID`, `clone RUN_ID`, diff --git a/src/cli.js b/src/cli.js index 19af96e..f0d5395 100644 --- a/src/cli.js +++ b/src/cli.js @@ -785,6 +785,7 @@ async function handleOptimization(client, command, positionals, parsed, io) { promotion_policy: firstValue(parsed.flags.promotionPolicy) || "manual", models: { eval_execution: firstValue(parsed.flags.evalModel) || sharedModel, + judge: firstValue(parsed.flags.judgeModel) || sharedModel, proposal: firstValue(parsed.flags.proposalModel) || sharedModel, analysis: firstValue(parsed.flags.analysisModel) || firstValue(parsed.flags.proposalModel) @@ -1646,6 +1647,7 @@ function normalizeFlagName(rawName) { "--review-policy": "reviewPolicy", "--promotion-policy": "promotionPolicy", "--eval-model": "evalModel", + "--judge-model": "judgeModel", "--analysis-model": "analysisModel", "--proposal-model": "proposalModel", "--base-hash": "baseHash", @@ -1904,7 +1906,8 @@ Data products: and --use-semantic-layer or --no-semantic-layer Optimization: answerlayer optimize start --name NAME --connection ID --suite ID --model MODEL - [--eval-model MODEL] [--proposal-model MODEL] [--analysis-model MODEL] + [--eval-model MODEL] [--judge-model MODEL] [--analysis-model MODEL] + [--proposal-model MODEL] [--mode recommendation|supervised|fully_automatic] [--component entities,relationships,measures,metrics,dimensions,filters] [--review-policy every_sweep|exceptions_only|end_of_run] diff --git a/test/cli.test.js b/test/cli.test.js index 23c20d9..87e3636 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -1071,6 +1071,7 @@ test("optimize start freezes explicit policies, budgets, models, and selectors", assert.equal(body.promotion_policy, "manual"); assert.equal(body.models.analysis, "openai.gpt-5.6-luna"); assert.equal(body.models.proposal, "openai.gpt-5.6-terra"); + assert.equal(body.models.judge, "openai.judge-model"); assert.equal(body.budget.max_iterations, 4); return new Response(JSON.stringify({ id: "run-1", execution_status: "queued" }), { status: 201, @@ -1087,6 +1088,7 @@ test("optimize start freezes explicit policies, budgets, models, and selectors", "--connection", "connection-1", "--suite", "suite-1", "--model", "openai.gpt-5.6-terra", + "--judge-model", "openai.judge-model", "--analysis-model", "openai.gpt-5.6-luna", "--mode", "fully_automatic", "--review-policy", "end_of_run", @@ -1111,6 +1113,7 @@ test("optimize start defaults analysis model to the explicit proposal model", as assert.equal(body.models.eval_execution, "openai.eval-model"); assert.equal(body.models.proposal, "openai.proposal-model"); assert.equal(body.models.analysis, "openai.proposal-model"); + assert.equal(body.models.judge, undefined); return new Response(JSON.stringify({ id: "run-1" }), { status: 201, headers: { "content-type": "application/json" }, @@ -1141,6 +1144,7 @@ test("optimize start defaults analysis model to the shared model", async () => { assert.equal(body.models.eval_execution, "openai.shared-model"); assert.equal(body.models.proposal, "openai.shared-model"); assert.equal(body.models.analysis, "openai.shared-model"); + assert.equal(body.models.judge, "openai.shared-model"); return new Response(JSON.stringify({ id: "run-1" }), { status: 201, headers: { "content-type": "application/json" },