diff --git a/.gittensory.yml.example b/.gittensory.yml.example index 51c61614ea..2cb246b6dc 100644 --- a/.gittensory.yml.example +++ b/.gittensory.yml.example @@ -899,7 +899,8 @@ settings: # e2eTestGen: false # Auto-generated E2E test scaffolding. Default: false. # planner: false # The @gittensory plan / issue-planner completion. Default: false. # summaries: false # AI summaries/rewrite text. Default: false. - # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-ONLY (never the frontier env.AI); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. + # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-first (never the frontier env.AI unless chatQaFrontierFallback below is also true); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. + # chatQaFrontierFallback: false # Opt-in only: falls back to the frontier env.AI chain if env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless chatQa is also true. Default: false. # Maintainer AI review tuning (`.gittensory.yml` top-level `review:` block). These knobs shape the advisory AI # review prompt and file selection only — gate/slop/secret-scan are unaffected. diff --git a/apps/gittensory-ui/public/openapi.json b/apps/gittensory-ui/public/openapi.json index d7e1d7a103..d1b00c33d2 100644 --- a/apps/gittensory-ui/public/openapi.json +++ b/apps/gittensory-ui/public/openapi.json @@ -9170,7 +9170,11 @@ }, "chatQa": { "type": "boolean", - "description": "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-only: unlike the four capabilities above it NEVER falls back to the frontier env.AI when off; it declines unless this is true AND env.AI_ADVISORY is configured. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command." + "description": "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-first: unlike the four capabilities above, it declines instead of falling back to the frontier env.AI when env.AI_ADVISORY is unconfigured, unless `chatQaFrontierFallback` is also enabled. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command." + }, + "chatQaFrontierFallback": { + "type": "boolean", + "description": "Opt-in only (#4595 follow-up): when true, `@gittensory chat` falls back to the shared frontier env.AI chain if env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless `chatQa` is also true. Default false -- a self-hoster without a local GPU may enable this to use their own frontier subscription/tokens for chat instead." } }, "required": [ @@ -9178,7 +9182,8 @@ "e2eTestGen", "planner", "summaries", - "chatQa" + "chatQa", + "chatQaFrontierFallback" ] }, "gittensorLabel": { diff --git a/config/examples/gittensory.full.yml b/config/examples/gittensory.full.yml index b749475499..53ac17db39 100644 --- a/config/examples/gittensory.full.yml +++ b/config/examples/gittensory.full.yml @@ -912,7 +912,8 @@ settings: # e2eTestGen: false # Auto-generated E2E test scaffolding. Default: false. # planner: false # The @gittensory plan / issue-planner completion. Default: false. # summaries: false # AI summaries/rewrite text. Default: false. - # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-ONLY (never the frontier env.AI); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. + # chatQa: false # @gittensory chat grounded LLM Q&A. Ollama-first (never the frontier env.AI unless chatQaFrontierFallback below is also true); needs env.AI_ADVISORY set. Co-requisite: commandRateLimitPolicy: hold (defaults off). Default: false. + # chatQaFrontierFallback: false # Opt-in only: falls back to the frontier env.AI chain if env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless chatQa is also true. Default: false. # Maintainer AI review tuning (`.gittensory.yml` top-level `review:` block). These knobs shape the advisory AI # review prompt and file selection only — gate/slop/secret-scan are unaffected. diff --git a/packages/gittensory-engine/src/focus-manifest.ts b/packages/gittensory-engine/src/focus-manifest.ts index 0b0114d302..8e84564e2f 100644 --- a/packages/gittensory-engine/src/focus-manifest.ts +++ b/packages/gittensory-engine/src/focus-manifest.ts @@ -1960,6 +1960,7 @@ function parseSettingsOverride(value: JsonValue | undefined, warnings: string[], if (typeof rawRouting.planner === "boolean") sparseRouting.planner = validated.planner; if (typeof rawRouting.summaries === "boolean") sparseRouting.summaries = validated.summaries; if (typeof rawRouting.chatQa === "boolean") sparseRouting.chatQa = validated.chatQa; + if (typeof rawRouting.chatQaFrontierFallback === "boolean") sparseRouting.chatQaFrontierFallback = validated.chatQaFrontierFallback; out.advisoryAiRouting = sparseRouting; } else if (r.advisoryAiRouting !== undefined) { warnings.push(`Manifest "settings.advisoryAiRouting" must be an object; ignoring it and keeping any existing policy.`); diff --git a/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts b/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts index 5ea588697f..ed90b1e54e 100644 --- a/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts +++ b/packages/gittensory-engine/src/review/advisory-ai-routing-config.ts @@ -6,6 +6,7 @@ export const DEFAULT_ADVISORY_AI_ROUTING: AdvisoryAiRoutingConfig = { planner: false, summaries: false, chatQa: false, + chatQaFrontierFallback: false, }; function normalizeField(value: unknown, field: keyof AdvisoryAiRoutingConfig, warnings: string[]): boolean { @@ -33,5 +34,6 @@ export function normalizeAdvisoryAiRoutingConfig(input: unknown, warnings: strin planner: normalizeField(record.planner, "planner", warnings), summaries: normalizeField(record.summaries, "summaries", warnings), chatQa: normalizeField(record.chatQa, "chatQa", warnings), + chatQaFrontierFallback: normalizeField(record.chatQaFrontierFallback, "chatQaFrontierFallback", warnings), }; } diff --git a/packages/gittensory-engine/src/types/manifest-deps-types.ts b/packages/gittensory-engine/src/types/manifest-deps-types.ts index 7ea7e43318..5b7ade1acd 100644 --- a/packages/gittensory-engine/src/types/manifest-deps-types.ts +++ b/packages/gittensory-engine/src/types/manifest-deps-types.ts @@ -112,9 +112,13 @@ export type AdvisoryAiRoutingConfig = { e2eTestGen: boolean; planner: boolean; summaries: boolean; - /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-only: unlike the four fields above it NEVER - * falls back to the frontier env.AI when off -- it simply declines. Default false. */ + /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-first: declines when off or when + * env.AI_ADVISORY is unconfigured and {@link chatQaFrontierFallback} is not also enabled. Default false. */ chatQa: boolean; + /** Opt-in ONLY (#4595 follow-up): when true, chat falls back to the shared frontier env.AI chain if + * env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless {@link chatQa} is also true. + * Default false. */ + chatQaFrontierFallback: boolean; }; export type ContributorBlacklistEntry = { diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index b6454f1e42..3f31d79c62 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -762,7 +762,12 @@ export const RepositorySettingsSchema = z chatQa: z .boolean() .describe( - "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-only: unlike the four capabilities above it NEVER falls back to the frontier env.AI when off; it declines unless this is true AND env.AI_ADVISORY is configured. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command.", + "Opt the `@gittensory chat ` grounded Q&A surface (#4595) into local Ollama inference. Ollama-first: unlike the four capabilities above, it declines instead of falling back to the frontier env.AI when env.AI_ADVISORY is unconfigured, unless `chatQaFrontierFallback` is also enabled. Co-requisite: set `commandRateLimitPolicy` to `hold` (it defaults to `off` fleet-wide) so the tighter `commandRateLimitAiMaxPerWindow` ceiling actually throttles this cost-bearing command.", + ), + chatQaFrontierFallback: z + .boolean() + .describe( + "Opt-in only (#4595 follow-up): when true, `@gittensory chat` falls back to the shared frontier env.AI chain if env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless `chatQa` is also true. Default false -- a self-hoster without a local GPU may enable this to use their own frontier subscription/tokens for chat instead.", ), }) .optional(), diff --git a/src/review/advisory-ai-routing-config.ts b/src/review/advisory-ai-routing-config.ts index e6032cbedf..1be773e972 100644 --- a/src/review/advisory-ai-routing-config.ts +++ b/src/review/advisory-ai-routing-config.ts @@ -6,6 +6,7 @@ export const DEFAULT_ADVISORY_AI_ROUTING: AdvisoryAiRoutingConfig = { planner: false, summaries: false, chatQa: false, + chatQaFrontierFallback: false, }; function normalizeField(value: unknown, field: keyof AdvisoryAiRoutingConfig, warnings: string[]): boolean { @@ -33,5 +34,6 @@ export function normalizeAdvisoryAiRoutingConfig(input: unknown, warnings: strin planner: normalizeField(record.planner, "planner", warnings), summaries: normalizeField(record.summaries, "summaries", warnings), chatQa: normalizeField(record.chatQa, "chatQa", warnings), + chatQaFrontierFallback: normalizeField(record.chatQaFrontierFallback, "chatQaFrontierFallback", warnings), }; } diff --git a/src/services/ai-chat-qa.ts b/src/services/ai-chat-qa.ts index 67ec63f54f..ad2def9d6b 100644 --- a/src/services/ai-chat-qa.ts +++ b/src/services/ai-chat-qa.ts @@ -13,11 +13,12 @@ import type { AgentRunBundle } from "./agent-orchestrator"; // ALREADY-deterministic decision-pack facts in the bundle (PR verdict, which checks/findings are blocking, // what a finding means) into natural prose; it never synthesizes new claims. // -// Ollama-ONLY, by hard requirement (#4595 requirement 5): unlike the four sibling advisoryAiRouting -// capabilities (slop/e2eTestGen/planner/summaries), which silently fall back to the shared frontier env.AI when -// their flag is off, this surface NEVER touches the frontier. It declines whenever advisoryAiRouting.chatQa is -// not true or env.AI_ADVISORY is unconfigured -- it does not call withAdvisoryAiEnv(env, false) and let a -// frontier token be spent. +// Ollama-FIRST by default (#4595 requirement 5): unlike the four sibling advisoryAiRouting capabilities +// (slop/e2eTestGen/planner/summaries), which silently fall back to the shared frontier env.AI whenever their +// OWN flag is off, this surface declines instead of spending a frontier token when env.AI_ADVISORY is +// unconfigured -- UNLESS the operator explicitly opts into advisoryAiRouting.chatQaFrontierFallback (a +// self-hoster without a local GPU may prefer their own frontier subscription over declining outright). Default +// false preserves the original Ollama-only behavior for every existing deployment. export type ChatQaResult = | { status: "disabled"; reason: string } @@ -80,14 +81,21 @@ type ChatGroundingBundle = { }; export async function generateChatQaAnswer(env: Env, req: ChatQaRequest): Promise { - // (#4595 req 5) Ollama-only enablement. BOTH gates are hard declines, never a frontier fallback. if (req.advisoryAiRouting?.chatQa !== true) { return { status: "disabled", reason: "Chat Q&A is not enabled on this instance (settings.advisoryAiRouting.chatQa is off)." }; } - if (!env.AI_ADVISORY) { + // Ollama-first: env.AI_ADVISORY is always preferred. env.AI (frontier) is only ever touched when the + // operator has explicitly opted in via advisoryAiRouting.chatQaFrontierFallback (#4595 follow-up) -- + // otherwise this stays exactly as Ollama-only as it originally shipped. + const frontierFallbackAllowed = req.advisoryAiRouting?.chatQaFrontierFallback === true; + const ai = env.AI_ADVISORY ?? (frontierFallbackAllowed ? env.AI : undefined); + const usedFrontier = !env.AI_ADVISORY && ai !== undefined; + if (!ai) { return { status: "unavailable", - reason: "Local advisory inference (env.AI_ADVISORY) is not configured; chat Q&A never falls back to the frontier model.", + reason: frontierFallbackAllowed + ? "Neither local advisory inference (env.AI_ADVISORY) nor the frontier model (env.AI) is configured." + : "Local advisory inference (env.AI_ADVISORY) is not configured; chat Q&A does not fall back to the frontier model unless advisoryAiRouting.chatQaFrontierFallback is enabled.", }; } @@ -134,12 +142,13 @@ export async function generateChatQaAnswer(env: Env, req: ChatQaRequest): Promis status: "quota_exceeded", estimatedNeurons: 0, detail: `estimated ${estimatedNeurons} neurons exceeds remaining budget ${remainingBudget}`, + usedFrontier, }); return { status: "quota_exceeded", model, estimatedNeurons, remainingBudget }; } try { - const response = await env.AI_ADVISORY.run(model, { + const response = await ai.run(model, { messages: [ { role: "system", content: CHAT_QA_SYSTEM_PROMPT }, { role: "user", content: prompt }, @@ -150,14 +159,14 @@ export async function generateChatQaAnswer(env: Env, req: ChatQaRequest): Promis const rawText = extractAiText(response); if (!rawText) throw new Error("empty_chat_answer"); if (containsPublicForbiddenText(rawText)) { - await recordChatAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "chat answer failed public sanitizer" }); + await recordChatAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "chat answer failed public sanitizer", usedFrontier }); return { status: "unsafe", model, estimatedNeurons, reason: "chat answer failed public sanitizer" }; } - await recordChatAi(env, req, { model, status: "ok", estimatedNeurons, detail: "chat answer generated" }); + await recordChatAi(env, req, { model, status: "ok", estimatedNeurons, detail: "chat answer generated", usedFrontier }); return { status: "ok", model, estimatedNeurons, text: rawText.trim() }; } catch (error) { const reason = error instanceof Error ? error.message : "chat_answer_failed"; - await recordChatAi(env, req, { model, status: "error", estimatedNeurons: 0, detail: reason }); + await recordChatAi(env, req, { model, status: "error", estimatedNeurons: 0, detail: reason, usedFrontier }); return { status: "error", model, estimatedNeurons, reason }; } } @@ -240,7 +249,7 @@ function auditOutcomeForAiStatus(status: string): "success" | "denied" | "error" async function recordChatAi( env: Env, req: ChatQaRequest, - event: { model: string; status: string; estimatedNeurons: number; detail: string }, + event: { model: string; status: string; estimatedNeurons: number; detail: string; usedFrontier: boolean }, ): Promise { await recordAiUsageEvent(env, { feature: "chat_qa", @@ -250,7 +259,9 @@ async function recordChatAi( status: event.status, estimatedNeurons: event.estimatedNeurons, detail: event.detail, - metadata: { repoFullName: req.repoFullName, issueNumber: req.issueNumber }, + // provider is observability-only: lets an operator who opted into chatQaFrontierFallback see, per event, + // whether a given answer actually spent a frontier token or stayed on the (free) local GPU. + metadata: { repoFullName: req.repoFullName, issueNumber: req.issueNumber, provider: event.usedFrontier ? "frontier" : "advisory" }, }); await recordAuditEvent(env, { eventType: "ai.chat_qa", @@ -258,7 +269,13 @@ async function recordChatAi( route: req.route, outcome: auditOutcomeForAiStatus(event.status), detail: event.detail, - metadata: { repoFullName: req.repoFullName, issueNumber: req.issueNumber, model: event.model, estimatedNeurons: event.estimatedNeurons }, + metadata: { + repoFullName: req.repoFullName, + issueNumber: req.issueNumber, + model: event.model, + estimatedNeurons: event.estimatedNeurons, + provider: event.usedFrontier ? "frontier" : "advisory", + }, }); } diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index e52c6cd4d2..cf509de45b 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -599,6 +599,7 @@ export function resolveEffectiveSettings( planner: advisoryAiRoutingOverride.planner ?? base.planner, summaries: advisoryAiRoutingOverride.summaries ?? base.summaries, chatQa: advisoryAiRoutingOverride.chatQa ?? base.chatQa, + chatQaFrontierFallback: advisoryAiRoutingOverride.chatQaFrontierFallback ?? base.chatQaFrontierFallback, }; } applyGateConfigOverrides(effective, manifest.gate); diff --git a/src/types.ts b/src/types.ts index d9235bc316..541bed4b73 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1281,18 +1281,25 @@ export type UnlinkedIssueGuardrailConfig = { * settings.advisoryAiRouting` (global default in shared/root config, per-repo override); defaults all-false * so an operator must deliberately opt each capability in. * - * `chatQa` (#4595) is the ONE capability that does NOT share the others' silent-frontier fallback: the four - * cost-optimizing capabilities above quietly fall back to the shared frontier env.AI when their flag is off, - * but the `@gittensory chat` grounded Q&A surface is "Ollama only" -- it declines/skips whenever - * `chatQa !== true` or `env.AI_ADVISORY` is unconfigured rather than ever spending a frontier token. */ + * `chatQa` (#4595) is the ONE capability that does NOT share the others' silent-frontier fallback BY DEFAULT: + * the four cost-optimizing capabilities above quietly fall back to the shared frontier env.AI when their flag + * is off, but the `@gittensory chat` grounded Q&A surface declines/skips whenever `chatQa !== true` or + * `env.AI_ADVISORY` is unconfigured, rather than ever spending a frontier token -- UNLESS `chatQaFrontierFallback` + * is also explicitly enabled (a self-hoster without a local GPU may prefer their own frontier subscription + * over an outright decline). */ export type AdvisoryAiRoutingConfig = { slop: boolean; e2eTestGen: boolean; planner: boolean; summaries: boolean; - /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-only: unlike the four fields above it NEVER - * falls back to the frontier env.AI when off -- it simply declines. Default false. */ + /** Grounded `@gittensory chat ` LLM Q&A (#4595). Ollama-first: declines when off or when + * env.AI_ADVISORY is unconfigured and {@link chatQaFrontierFallback} is not also enabled. Default false. */ chatQa: boolean; + /** Opt-in ONLY (#4595 follow-up): when true, chat falls back to the shared frontier env.AI chain if + * env.AI_ADVISORY is unconfigured, instead of declining. Meaningless unless {@link chatQa} is also true. + * Default false -- preserves the original Ollama-only behavior for every existing deployment; a self-hoster + * without a local GPU may enable this to use their own frontier subscription/tokens for chat instead. */ + chatQaFrontierFallback: boolean; }; /** A blocked contributor (#1425, anti-abuse): a GitHub `login` plus optional maintainer metadata. The converged diff --git a/test/unit/advisory-ai-routing-call-sites.test.ts b/test/unit/advisory-ai-routing-call-sites.test.ts index d0aa430882..54eca10a2d 100644 --- a/test/unit/advisory-ai-routing-call-sites.test.ts +++ b/test/unit/advisory-ai-routing-call-sites.test.ts @@ -35,7 +35,7 @@ describe("runAiSlopForAdvisory routes through AI_ADVISORY (#4364)", () => { }); await runAiSlopForAdvisory(env, { mode: "live", - settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }), + settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false, chatQaFrontierFallback: false }), advisory, repoFullName: "owner/repo", pr: { number: 1, title: "t" }, @@ -80,7 +80,7 @@ describe("runAiSlopForAdvisory routes through AI_ADVISORY (#4364)", () => { const env = createTestEnv({ AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI: { run: frontierRun } as unknown as Ai }); await runAiSlopForAdvisory(env, { mode: "live", - settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }), + settings: settingsFixture({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false, chatQaFrontierFallback: false }), advisory, repoFullName: "owner/repo", pr: { number: 3, title: "t" }, diff --git a/test/unit/advisory-ai-routing-config-engine.test.ts b/test/unit/advisory-ai-routing-config-engine.test.ts index 752b7058b3..54cba24847 100644 --- a/test/unit/advisory-ai-routing-config-engine.test.ts +++ b/test/unit/advisory-ai-routing-config-engine.test.ts @@ -11,28 +11,34 @@ describe("normalizeAdvisoryAiRoutingConfig", () => { it("normalizes a fully-valid config", () => { const warnings: string[] = []; - expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true }, warnings)).toEqual({ + expect( + normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true, chatQaFrontierFallback: true }, warnings), + ).toEqual({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true, + chatQaFrontierFallback: true, }); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("defaults %s to false when omitted", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa", "chatQaFrontierFallback"] as const)("defaults %s to false when omitted", (field) => { const warnings: string[] = []; expect(normalizeAdvisoryAiRoutingConfig({}, warnings)[field]).toBe(false); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("falls back to false and warns on a non-boolean %s", (field) => { - const warnings: string[] = []; - const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); - expect(cfg[field]).toBe(false); - expect(warnings).toEqual([`settings.advisoryAiRouting.${field} must be a boolean; using the default "false".`]); - }); + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa", "chatQaFrontierFallback"] as const)( + "falls back to false and warns on a non-boolean %s", + (field) => { + const warnings: string[] = []; + const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); + expect(cfg[field]).toBe(false); + expect(warnings).toEqual([`settings.advisoryAiRouting.${field} must be a boolean; using the default "false".`]); + }, + ); it.each([ ["an array", []], diff --git a/test/unit/advisory-ai-routing-config.test.ts b/test/unit/advisory-ai-routing-config.test.ts index d9db0f038d..6b7bb10426 100644 --- a/test/unit/advisory-ai-routing-config.test.ts +++ b/test/unit/advisory-ai-routing-config.test.ts @@ -10,33 +10,39 @@ describe("normalizeAdvisoryAiRoutingConfig", () => { it("normalizes a fully-valid config", () => { const warnings: string[] = []; - expect(normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true }, warnings)).toEqual({ + expect( + normalizeAdvisoryAiRoutingConfig({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true, chatQaFrontierFallback: true }, warnings), + ).toEqual({ slop: true, e2eTestGen: true, planner: true, summaries: true, chatQa: true, + chatQaFrontierFallback: true, }); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("defaults %s to false when omitted", (field) => { + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa", "chatQaFrontierFallback"] as const)("defaults %s to false when omitted", (field) => { const warnings: string[] = []; expect(normalizeAdvisoryAiRoutingConfig({}, warnings)[field]).toBe(false); expect(warnings).toEqual([]); }); - it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa"] as const)("falls back to false and warns on a non-boolean %s", (field) => { - const warnings: string[] = []; - const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); - expect(cfg[field]).toBe(false); - expect(warnings).toEqual([`settings.advisoryAiRouting.${field} must be a boolean; using the default "false".`]); - }); + it.each(["slop", "e2eTestGen", "planner", "summaries", "chatQa", "chatQaFrontierFallback"] as const)( + "falls back to false and warns on a non-boolean %s", + (field) => { + const warnings: string[] = []; + const cfg = normalizeAdvisoryAiRoutingConfig({ [field]: "yes" }, warnings); + expect(cfg[field]).toBe(false); + expect(warnings).toEqual([`settings.advisoryAiRouting.${field} must be a boolean; using the default "false".`]); + }, + ); it("normalizes one valid field alongside one invalid field independently", () => { const warnings: string[] = []; const cfg = normalizeAdvisoryAiRoutingConfig({ slop: true, planner: "nope" }, warnings); - expect(cfg).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false }); + expect(cfg).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: false, chatQaFrontierFallback: false }); expect(warnings).toEqual([`settings.advisoryAiRouting.planner must be a boolean; using the default "false".`]); }); diff --git a/test/unit/ai-chat-qa.test.ts b/test/unit/ai-chat-qa.test.ts index 1530f64ee4..0047ba2222 100644 --- a/test/unit/ai-chat-qa.test.ts +++ b/test/unit/ai-chat-qa.test.ts @@ -3,8 +3,9 @@ import { __chatQaInternals, CHAT_QA_FALLBACK_COMMAND, generateChatQaAnswer } fro import type { AgentRunBundle } from "../../src/services/agent-orchestrator"; import { createTestEnv } from "../helpers/d1"; -const ADVISORY_ON = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true }; -const ADVISORY_OFF = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false }; +const ADVISORY_ON = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: false }; +const ADVISORY_OFF = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false, chatQaFrontierFallback: false }; +const ADVISORY_ON_FRONTIER_FALLBACK = { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: true }; function bundleFixture(runOverrides?: Partial, actionOverrides?: Partial): AgentRunBundle { return { @@ -79,7 +80,7 @@ describe("generateChatQaAnswer", () => { expect(result.status).toBe("disabled"); }); - it("never falls back to the frontier chain: reports unavailable when chatQa is on but AI_ADVISORY is unconfigured", async () => { + it("by default (chatQaFrontierFallback off) never falls back to the frontier chain: reports unavailable when chatQa is on but AI_ADVISORY is unconfigured", async () => { const frontierRun = vi.fn(); const env = createTestEnv({ AI: { run: frontierRun } as unknown as Ai }); const result = await generateChatQaAnswer(env, { @@ -90,9 +91,57 @@ describe("generateChatQaAnswer", () => { issueNumber: 1, }); expect(result).toMatchObject({ status: "unavailable" }); + expect(result.status === "unavailable" ? result.reason : "").toContain("does not fall back to the frontier model"); expect(frontierRun).not.toHaveBeenCalled(); }); + it("#4595 follow-up: falls back to the frontier chain when chatQaFrontierFallback is enabled and AI_ADVISORY is unconfigured", async () => { + const frontierRun = vi.fn(async () => ({ response: "Frontier-served answer." })); + const env = createTestEnv({ AI: { run: frontierRun } as unknown as Ai, AI_DAILY_NEURON_BUDGET: "10000" }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON_FRONTIER_FALLBACK, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "ok", text: "Frontier-served answer." }); + expect(frontierRun).toHaveBeenCalled(); + }); + + it("#4595 follow-up: still prefers AI_ADVISORY (Ollama) over the frontier chain even when chatQaFrontierFallback is enabled", async () => { + const advisoryRun = vi.fn(async () => ({ response: "Ollama-served answer." })); + const frontierRun = vi.fn(); + const env = createTestEnv({ + AI_ADVISORY: { run: advisoryRun } as unknown as Ai, + AI: { run: frontierRun } as unknown as Ai, + AI_DAILY_NEURON_BUDGET: "10000", + }); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON_FRONTIER_FALLBACK, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "ok", text: "Ollama-served answer." }); + expect(advisoryRun).toHaveBeenCalled(); + expect(frontierRun).not.toHaveBeenCalled(); + }); + + it("#4595 follow-up: reports unavailable with a distinct message when chatQaFrontierFallback is enabled but NEITHER provider is configured", async () => { + const env = createTestEnv({}); + const result = await generateChatQaAnswer(env, { + bundle: bundleFixture(), + question: "why is this blocked?", + advisoryAiRouting: ADVISORY_ON_FRONTIER_FALLBACK, + repoFullName: "owner/repo", + issueNumber: 1, + }); + expect(result).toMatchObject({ status: "unavailable" }); + expect(result.status === "unavailable" ? result.reason : "").toContain("Neither local advisory inference"); + }); + it("declines when no question is supplied", async () => { const advisoryRun = vi.fn(); const env = createTestEnv({ AI_ADVISORY: { run: advisoryRun } as unknown as Ai }); diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index c3f5520d77..6f12e31ce1 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -3048,7 +3048,7 @@ describe("parseFocusManifest settings override + resolveEffectiveSettings", () = it("resolveEffectiveSettings falls back to the all-off built-in default when the DB layer has no advisoryAiRouting at all (#4364)", () => { const db = {} as unknown as RepositorySettings; const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { planner: true } } })); - expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: true, summaries: false, chatQa: false }); + expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: true, summaries: false, chatQa: false, chatQaFrontierFallback: false }); }); it("wires settings.advisoryAiRouting.chatQa into the manifest parser as a sparse override (#4595)", () => { @@ -3058,15 +3058,41 @@ describe("parseFocusManifest settings override + resolveEffectiveSettings", () = }); it("resolveEffectiveSettings merges an explicit chatQa override over the DB layer's value (#4595)", () => { - const db = { advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false } } as unknown as RepositorySettings; + const db = { + advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: false, chatQaFrontierFallback: false }, + } as unknown as RepositorySettings; const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { chatQa: true } } })); - expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true }); + expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: false }); }); it("resolveEffectiveSettings keeps the DB layer's chatQa when the manifest override omits it (#4595)", () => { - const db = { advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true } } as unknown as RepositorySettings; + const db = { + advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: false }, + } as unknown as RepositorySettings; + const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { slop: true } } })); + expect(eff.advisoryAiRouting).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: false }); + }); + + it("wires settings.advisoryAiRouting.chatQaFrontierFallback into the manifest parser as a sparse override (#4595 follow-up)", () => { + const parsed = parseFocusManifest({ settings: { advisoryAiRouting: { chatQaFrontierFallback: true } } }); + expect(parsed.settings.advisoryAiRouting).toEqual({ chatQaFrontierFallback: true }); + expect(parsed.warnings).toEqual([]); + }); + + it("resolveEffectiveSettings merges an explicit chatQaFrontierFallback override over the DB layer's value (#4595 follow-up)", () => { + const db = { + advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: false }, + } as unknown as RepositorySettings; + const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { chatQaFrontierFallback: true } } })); + expect(eff.advisoryAiRouting).toEqual({ slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: true }); + }); + + it("resolveEffectiveSettings keeps the DB layer's chatQaFrontierFallback when the manifest override omits it (#4595 follow-up)", () => { + const db = { + advisoryAiRouting: { slop: false, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: true }, + } as unknown as RepositorySettings; const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { advisoryAiRouting: { slop: true } } })); - expect(eff.advisoryAiRouting).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: true }); + expect(eff.advisoryAiRouting).toEqual({ slop: true, e2eTestGen: false, planner: false, summaries: false, chatQa: true, chatQaFrontierFallback: true }); }); it("drops a malformed advisoryAiRouting.slop field instead of replacing existing policy with defaults (#4364)", () => {