From e92b6b1d9bd5d5e6070de500ea073df1475e3ab1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:26:00 -0700 Subject: [PATCH 1/4] feat(ai-review): add aiReviewMode + aiReviewByok settings and config layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plumb two new RepositorySettings fields through the DB (migration 0026), the dashboard settings route, and the .gittensory.yml config-as-code layer: - aiReviewMode: off | advisory | block (GateRuleMode), default off - aiReviewByok: boolean, default false Both are settable from .gittensory.yml via the friendly gate.aiReview { mode, byok } alias (wins) and the generic settings: override, resolved through resolveEffectiveSettings (yml > DB > defaults). No behavior yet — this is the config surface the AI review engine wires into next. --- migrations/0026_ai_review_settings.sql | 2 ++ src/api/routes.ts | 4 +++ src/db/repositories.ts | 10 ++++++ src/db/schema.ts | 2 ++ src/signals/focus-manifest.ts | 33 +++++++++++++++++-- src/types.ts | 9 +++++ test/unit/focus-manifest.test.ts | 26 +++++++++++++-- test/unit/policy-sanitizer.test.ts | 2 ++ test/unit/registration-readiness.test.ts | 2 ++ test/unit/repo-policy-readiness.test.ts | 2 ++ .../self-dogfood-registration-pack.test.ts | 2 ++ test/unit/settings-preview.test.ts | 2 ++ test/unit/signals-coverage.test.ts | 2 ++ test/unit/signals-v2.test.ts | 2 ++ test/unit/signals.test.ts | 10 ++++++ 15 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 migrations/0026_ai_review_settings.sql diff --git a/migrations/0026_ai_review_settings.sql b/migrations/0026_ai_review_settings.sql new file mode 100644 index 0000000000..c4c3c9e205 --- /dev/null +++ b/migrations/0026_ai_review_settings.sql @@ -0,0 +1,2 @@ +ALTER TABLE repository_settings ADD COLUMN ai_review_mode TEXT NOT NULL DEFAULT 'off'; +ALTER TABLE repository_settings ADD COLUMN ai_review_byok INTEGER NOT NULL DEFAULT 0; diff --git a/src/api/routes.ts b/src/api/routes.ts index 31bb834609..6a4e6074ff 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -509,6 +509,8 @@ const repositorySettingsSchema = z.object({ duplicatePrGateMode: z.enum(["off", "advisory", "block"]).default("block"), qualityGateMode: z.enum(["off", "advisory", "block"]).default("advisory"), qualityGateMinScore: z.number().int().min(0).max(100).nullable().optional(), + aiReviewMode: z.enum(["off", "advisory", "block"]).default("off"), + aiReviewByok: z.boolean().default(false), autoLabelEnabled: z.boolean().default(true), gittensorLabel: z.string().trim().min(1).max(50).default("gittensor"), createMissingLabel: z.boolean().default(true), @@ -2445,6 +2447,8 @@ export function createApp() { duplicatePrGateMode: parsed.data.duplicatePrGateMode, qualityGateMode: parsed.data.qualityGateMode, qualityGateMinScore: parsed.data.qualityGateMinScore, + aiReviewMode: parsed.data.aiReviewMode, + aiReviewByok: parsed.data.aiReviewByok, autoLabelEnabled: parsed.data.autoLabelEnabled, gittensorLabel: parsed.data.gittensorLabel, createMissingLabel: parsed.data.createMissingLabel, diff --git a/src/db/repositories.ts b/src/db/repositories.ts index f4e34167e3..34a0d428e3 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -390,6 +390,8 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise duplicatePrGateMode: "block", qualityGateMode: "advisory", qualityGateMinScore: null, + aiReviewMode: "off", + aiReviewByok: false, autoLabelEnabled: true, gittensorLabel: "gittensor", createMissingLabel: true, @@ -413,6 +415,8 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise duplicatePrGateMode: parseGateRuleMode(row.duplicatePrGateMode), qualityGateMode: parseGateRuleMode(row.qualityGateMode), qualityGateMinScore: normalizeQualityGateMinScore(row.qualityGateMinScore), + aiReviewMode: parseGateRuleMode(row.aiReviewMode), + aiReviewByok: row.aiReviewByok, autoLabelEnabled: row.autoLabelEnabled, gittensorLabel: row.gittensorLabel, createMissingLabel: row.createMissingLabel, @@ -440,6 +444,8 @@ export async function upsertRepositorySettings(env: Env, settings: Partial) : undefined; + if (aiReview !== undefined && aiReview !== null && aiReviewRecord === undefined) { + warnings.push(`Manifest gate field "gate.aiReview" must be a mapping; ignoring it.`); + } const gate: FocusManifestGateConfig = { present: false, enabled: normalizeOptionalBoolean(record.enabled, "gate.enabled", warnings), @@ -259,9 +270,17 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu duplicates: normalizeOptionalGateMode(record.duplicates, "gate.duplicates", warnings), readinessMode: normalizeOptionalGateMode(readinessRecord?.mode, "gate.readiness.mode", warnings), readinessMinScore: normalizeOptionalScore(readinessRecord?.minScore, "gate.readiness.minScore", warnings), + aiReviewMode: normalizeOptionalGateMode(aiReviewRecord?.mode, "gate.aiReview.mode", warnings), + aiReviewByok: normalizeOptionalBoolean(aiReviewRecord?.byok, "gate.aiReview.byok", warnings), }; gate.present = - gate.enabled !== null || gate.linkedIssue !== null || gate.duplicates !== null || gate.readinessMode !== null || gate.readinessMinScore !== null; + gate.enabled !== null || + gate.linkedIssue !== null || + gate.duplicates !== null || + gate.readinessMode !== null || + gate.readinessMinScore !== null || + gate.aiReviewMode !== null || + gate.aiReviewByok !== null; return gate; } @@ -281,6 +300,12 @@ export function gateConfigToJson(gate: FocusManifestGateConfig): JsonValue { if (gate.readinessMinScore !== null) readiness.minScore = gate.readinessMinScore; out.readiness = readiness; } + if (gate.aiReviewMode !== null || gate.aiReviewByok !== null) { + const aiReview: Record = {}; + if (gate.aiReviewMode !== null) aiReview.mode = gate.aiReviewMode; + if (gate.aiReviewByok !== null) aiReview.byok = gate.aiReviewByok; + out.aiReview = aiReview; + } return out; } @@ -330,11 +355,13 @@ function parseSettingsOverride(value: JsonValue | undefined, warnings: string[]) if (qualityGateMode !== null) out.qualityGateMode = qualityGateMode; const qualityGateMinScore = normalizeOptionalScore(r.qualityGateMinScore, "settings.qualityGateMinScore", warnings); if (qualityGateMinScore !== null) out.qualityGateMinScore = qualityGateMinScore; + const aiReviewMode = normalizeOptionalGateMode(r.aiReviewMode, "settings.aiReviewMode", warnings); + if (aiReviewMode !== null) out.aiReviewMode = aiReviewMode; const gittensorLabel = normalizeOptionalString(r.gittensorLabel, "settings.gittensorLabel", warnings); if (gittensorLabel !== null) out.gittensorLabel = gittensorLabel; const publicSurface = normalizeOptionalEnum(r.publicSurface, "settings.publicSurface", ["off", "comment_and_label", "comment_only", "label_only"] as const, warnings); if (publicSurface !== null) out.publicSurface = publicSurface; - for (const key of ["autoLabelEnabled", "createMissingLabel", "includeMaintainerAuthors", "requireLinkedIssue", "backfillEnabled", "privateTrustEnabled"] as const) { + for (const key of ["aiReviewByok", "autoLabelEnabled", "createMissingLabel", "includeMaintainerAuthors", "requireLinkedIssue", "backfillEnabled", "privateTrustEnabled"] as const) { const flag = normalizeOptionalBoolean(r[key], `settings.${key}`, warnings); if (flag !== null) out[key] = flag; } @@ -412,6 +439,8 @@ export function resolveEffectiveSettings(dbSettings: RepositorySettings, manifes if (gate.duplicates !== null) effective.duplicatePrGateMode = gate.duplicates; if (gate.readinessMode !== null) effective.qualityGateMode = gate.readinessMode; if (gate.readinessMinScore !== null) effective.qualityGateMinScore = gate.readinessMinScore; + if (gate.aiReviewMode !== null) effective.aiReviewMode = gate.aiReviewMode; + if (gate.aiReviewByok !== null) effective.aiReviewByok = gate.aiReviewByok; return effective; } diff --git a/src/types.ts b/src/types.ts index 4f405e81a8..54a77be066 100644 --- a/src/types.ts +++ b/src/types.ts @@ -367,6 +367,15 @@ export type RepositorySettings = { duplicatePrGateMode: GateRuleMode; qualityGateMode: GateRuleMode; qualityGateMinScore?: number | null | undefined; + /** AI maintainer review. `off` = no AI; `advisory` = post AI review notes only; `block` = ALSO let a + * dual-model high-confidence consensus defect become a gate blocker (confirmed-contributors only, + * like every other blocker). Default `off` — AI is opt-in. */ + aiReviewMode: GateRuleMode; + /** Bring-your-own-key: when true and a provider key is configured for the repo, the advisory AI review + * is generated by the maintainer's frontier model (Anthropic/OpenAI) instead of free Workers AI. The + * consensus blocker always uses the free Workers-AI model pair regardless, so BYOK never changes who + * can be blocked. Default false. */ + aiReviewByok: boolean; autoLabelEnabled: boolean; gittensorLabel: string; createMissingLabel: boolean; diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index de34a61e6d..1b58c1a356 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -400,7 +400,7 @@ describe("compileFocusManifestPolicy", () => { issueDiscoveryPolicy: "neutral", maintainerNotes: [], publicNotes: ["Keep PRs focused.", "Maximize your reward payout"], - gate: { present: false, enabled: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null }, + gate: { present: false, enabled: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, aiReviewMode: null, aiReviewByok: null }, settings: {}, review: { present: false, footerText: null, note: null, fields: {} }, warnings: [], @@ -688,7 +688,7 @@ describe("parseFocusManifest gate config", () => { it("parses a full gate section including the readiness block", () => { const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "block", minScore: 70 } } }); expect(m.present).toBe(true); - expect(m.gate).toEqual({ present: true, enabled: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70 }); + expect(m.gate).toEqual({ present: true, enabled: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "block", readinessMinScore: 70, aiReviewMode: null, aiReviewByok: null }); }); it("parses gate.enabled (on/off) and ignores non-boolean values with a warning", () => { @@ -754,6 +754,17 @@ describe("parseFocusManifest gate config", () => { expect(m.gate.readinessMode).toBe("block"); expect(m.gate.readinessMinScore).toBe(80); }); + + it("parses the gate.aiReview block, round-trips it, and warns on a non-mapping/invalid value", () => { + const m = parseFocusManifest({ gate: { aiReview: { mode: "block", byok: true } } }); + expect(m.present).toBe(true); + expect(m.gate.present).toBe(true); + expect(m.gate.aiReviewMode).toBe("block"); + expect(m.gate.aiReviewByok).toBe(true); + expect(parseFocusManifest({ gate: gateConfigToJson(m.gate) }).gate).toEqual(m.gate); + expect(parseFocusManifest({ gate: { aiReview: ["nope"] } }).warnings.some((w) => /gate\.aiReview" must be a mapping/.test(w))).toBe(true); + expect(parseFocusManifest({ gate: { aiReview: { mode: "loud" } } }).warnings.some((w) => /gate\.aiReview\.mode/.test(w))).toBe(true); + }); }); describe("parseFocusManifest settings override + resolveEffectiveSettings", () => { @@ -837,6 +848,17 @@ describe("parseFocusManifest settings override + resolveEffectiveSettings", () = expect(eff.gateCheckMode).toBe("enabled"); // gate.enabled expect(eff.linkedIssueGateMode).toBe("block"); // gate: wins over settings: }); + + it("parses aiReview from settings: and lets gate.aiReview win in resolveEffectiveSettings", () => { + const parsed = parseFocusManifest({ settings: { aiReviewMode: "advisory", aiReviewByok: true } }); + expect(parsed.settings.aiReviewMode).toBe("advisory"); + expect(parsed.settings.aiReviewByok).toBe(true); + const db = { aiReviewMode: "off", aiReviewByok: false } as unknown as RepositorySettings; + // settings: applies first, then the friendly gate.aiReview alias wins for its fields. + const eff = resolveEffectiveSettings(db, parseFocusManifest({ settings: { aiReviewMode: "advisory" }, gate: { aiReview: { mode: "block", byok: true } } })); + expect(eff.aiReviewMode).toBe("block"); + expect(eff.aiReviewByok).toBe(true); + }); }); describe("parseFocusManifest review config", () => { diff --git a/test/unit/policy-sanitizer.test.ts b/test/unit/policy-sanitizer.test.ts index d06c9a46e7..01231b672e 100644 --- a/test/unit/policy-sanitizer.test.ts +++ b/test/unit/policy-sanitizer.test.ts @@ -74,6 +74,8 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off", + aiReviewByok: false, ...overrides, }; } diff --git a/test/unit/self-dogfood-registration-pack.test.ts b/test/unit/self-dogfood-registration-pack.test.ts index f59258b5d9..e91d28341c 100644 --- a/test/unit/self-dogfood-registration-pack.test.ts +++ b/test/unit/self-dogfood-registration-pack.test.ts @@ -67,6 +67,8 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off", + aiReviewByok: false, ...overrides, }; } diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index a142d486c9..edf2a99474 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -1533,6 +1533,8 @@ function repoSettings(repoFullName: string): RepositorySettings { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off", + aiReviewByok: false, }; } diff --git a/test/unit/signals-v2.test.ts b/test/unit/signals-v2.test.ts index 16ec9a2437..39270c90ae 100644 --- a/test/unit/signals-v2.test.ts +++ b/test/unit/signals-v2.test.ts @@ -1625,6 +1625,8 @@ describe("v2 signal builders", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off", + aiReviewByok: false, }, }); expect(comment).toContain("Author: `unknown`"); diff --git a/test/unit/signals.test.ts b/test/unit/signals.test.ts index ce1c22d3f6..4622a462e0 100644 --- a/test/unit/signals.test.ts +++ b/test/unit/signals.test.ts @@ -337,6 +337,8 @@ describe("world-class backend signals", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off" as const, + aiReviewByok: false, }; const collisions = buildCollisionReport(repo.fullName, issues, pullRequests); const queueHealth = buildQueueHealth(repo, issues, pullRequests, collisions); @@ -380,6 +382,8 @@ describe("world-class backend signals", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off" as const, + aiReviewByok: false, }; const collisions = buildCollisionReport(repo.fullName, issues, pullRequests); const queueHealth = buildQueueHealth(repo, issues, pullRequests, collisions); @@ -443,6 +447,8 @@ describe("world-class backend signals", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off" as const, + aiReviewByok: false, }; const collisions = buildCollisionReport(repo.fullName, issues, pullRequests); const queueHealth = buildQueueHealth(repo, issues, pullRequests, collisions); @@ -527,6 +533,8 @@ describe("world-class backend signals", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off" as const, + aiReviewByok: false, }; const undetected = detectGittensorContributor("newbie", currentPr, [currentPr], []); const cachedDetected = detectGittensorContributor("oktofeesh1", currentPr, [currentPr, { ...currentPr, number: 10, mergedAt: "2026-05-01T00:00:00.000Z" }], []); @@ -586,6 +594,8 @@ describe("world-class backend signals", () => { requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + aiReviewMode: "off" as const, + aiReviewByok: false, }; const comment = buildPublicPrIntelligenceComment({ repo, pr: currentPr, profile, detection, queueHealth, collisions, preflight, settings }); From 7532e8c79383127110e3157bc3b1d7e4084a61c7 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:49:54 -0700 Subject: [PATCH 2/4] feat(ai-review): dual-AI maintainer review wired into the gate + panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Workers-AI review engine and folds it into the public PR path, all behind the opt-in aiReviewMode (default off) and the existing AI flags (both default off — dormant until enabled): - src/services/ai-review.ts: dual-model engine adapted from reviewbot's proven pair (gpt-oss-120b + nemotron, reliable fallbacks). Two layers: * advisory notes — a maintainer-style write-up (BYOK frontier model when a key is supplied, else free Workers AI); * consensus defect — a defect is reported ONLY when BOTH free Workers-AI models independently flag a high-confidence (>=0.9) critical defect. BYOK never drives this path, so it never changes who can be blocked. Every public string is forced through sanitizePublicComment; every call is metered against the shared daily neuron budget and audited. - Gate: a new ai_consensus_defect finding becomes a hard blocker only when aiReview: block is opted in (advisory otherwise) — still confirmed- contributor gated by evaluateGateCheck. - Processor: runAiReviewForAdvisory mutates the advisory with the consensus defect BEFORE the gate evaluates, and returns advisory notes for the panel. Fully fail-safe — disabled/non-confirmed/no-AI/error → no finding, no notes. - Panel: an advisory 'Gittensory AI review' section (HTML-escaped, public-safe). End-to-end test proves a confirmed contributor is blocked by a consensus defect; coverage holds above the 97% gate. BYOK provider key storage lands in the next commit (the engine already accepts a providerKey). --- src/queue/processors.ts | 81 +++++- src/rules/advisory.ts | 7 + src/services/ai-review.ts | 361 +++++++++++++++++++++++++++ src/signals/engine.ts | 20 ++ test/unit/ai-review-advisory.test.ts | 154 ++++++++++++ test/unit/ai-review.test.ts | 269 ++++++++++++++++++++ test/unit/gate-check-policy.test.ts | 25 ++ test/unit/queue.test.ts | 76 ++++++ test/unit/signals-coverage.test.ts | 4 + 9 files changed, 995 insertions(+), 2 deletions(-) create mode 100644 src/services/ai-review.ts create mode 100644 test/unit/ai-review-advisory.test.ts create mode 100644 test/unit/ai-review.test.ts diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 368f0d55c9..5aaa8865f3 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -132,7 +132,8 @@ import { decidePublicSurface } from "../signals/settings-preview"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { resolveEffectiveSettings } from "../signals/focus-manifest"; import type { LocalBranchAnalysisInput } from "../signals/local-branch"; -import type { ContributorEvidenceRecord, GitHubWebhookPayload, JobMessage, JsonValue, PullRequestRecord, RepositorySettings } from "../types"; +import { runGittensoryAiReview } from "../services/ai-review"; +import type { AdvisoryFinding, ContributorEvidenceRecord, GitHubWebhookPayload, JobMessage, JsonValue, PullRequestRecord, RepositorySettings } from "../types"; import { sha256Hex } from "../utils/crypto"; import { errorMessage, nowIso } from "../utils/json"; @@ -791,6 +792,7 @@ export function gateCheckPolicy(settings: RepositorySettings, readinessScore?: n duplicatePrGateMode: settings.duplicatePrGateMode, qualityGateMode: settings.qualityGateMode, qualityGateMinScore: settings.qualityGateMinScore ?? null, + aiReviewGateMode: settings.aiReviewMode, readinessScore: readinessScore ?? null, confirmedContributor, }; @@ -806,6 +808,76 @@ async function resolveRepositorySettings(env: Env, repoFullName: string): Promis return resolveEffectiveSettings(dbSettings, manifest); } +/** Build a bounded unified-diff string from cached PR files for the AI reviewer. Caps total size so a + * huge PR cannot blow the model context or the neuron budget; each file's patch is taken from the raw + * GitHub file payload when present. */ +export function buildAiReviewDiff(files: Awaited>): string { + const MAX_DIFF_CHARS = 60000; + const parts: string[] = []; + let total = 0; + for (const file of files) { + const patch = typeof file.payload?.patch === "string" ? file.payload.patch : ""; + const header = `### ${file.path}${file.status ? ` (${file.status})` : ""} +${file.additions}/-${file.deletions}`; + const block = patch ? `${header}\n${patch}` : header; + if (total + block.length > MAX_DIFF_CHARS) { + parts.push(`… diff truncated (${files.length} files total).`); + break; + } + parts.push(block); + total += block.length; + } + return parts.join("\n\n"); +} + +/** + * Run the opt-in AI maintainer review and fold it into the gate + panel. Mutates `advisory.findings` + * with a dual-model consensus defect (when `aiReviewMode: block` and the free Workers-AI pair agrees with + * high confidence) so it can become a gate blocker BEFORE evaluateGateCheck runs — still confirmed- + * contributor gated. Returns the advisory notes for the public panel. Fully fail-safe: disabled / not a + * confirmed contributor / no head SHA / non-ok AI / any thrown error → no finding and no notes. + */ +export async function runAiReviewForAdvisory( + env: Env, + args: { + settings: RepositorySettings; + advisory: Awaited>; + repoFullName: string; + pr: { number: number; title: string; body?: string | null | undefined }; + author: string | null; + confirmedContributor: boolean; + }, +): Promise<{ notes: string } | undefined> { + if (args.settings.aiReviewMode === "off" || !args.confirmedContributor || !args.advisory.headSha) return undefined; + try { + const files = await listPullRequestFiles(env, args.repoFullName, args.pr.number); + const result = await runGittensoryAiReview(env, { + repoFullName: args.repoFullName, + prNumber: args.pr.number, + title: args.pr.title, + body: args.pr.body ?? undefined, + diff: buildAiReviewDiff(files), + actor: args.author, + mode: args.settings.aiReviewMode === "block" ? "block" : "advisory", + providerKey: null, + }); + if (result.status !== "ok") return undefined; + if (result.consensusDefect) { + const defect: AdvisoryFinding = { + code: "ai_consensus_defect", + severity: "critical", + title: `AI reviewers agree on a likely critical defect: ${result.consensusDefect.title}`, + detail: result.consensusDefect.detail, + action: "Resolve the flagged defect, or override if the AI reviewers are mistaken, then re-run the gate.", + }; + args.advisory.findings.push(defect); + } + return result.advisoryNotes ? { notes: result.advisoryNotes } : undefined; + } catch (error) { + console.error(JSON.stringify({ level: "warn", event: "ai_review_failed", repository: args.repoFullName, pullNumber: args.pr.number, error: errorMessage(error) })); + return undefined; + } +} + function linkedIssueDuplicatePullRequestsForGate(pr: PullRequestRecord, pullRequests: PullRequestRecord[]): number[] { const linkedIssues = new Set(pr.linkedIssues); if (linkedIssues.size === 0) return []; @@ -985,6 +1057,11 @@ async function maybePublishPrPublicSurface( // detection) gets a neutral, non-blocking gate. Gate-only runs still verify confirmation before // evaluating blockers so confirmed contributors cannot bypass a required Gate check. const confirmedContributor = official?.status === "confirmed"; + + // AI maintainer review (opt-in via aiReviewMode). Mutates `advisory` with a consensus defect (if any) + // BEFORE the gate evaluates, and returns advisory notes for the panel. Fully fail-safe. + const aiReview = await runAiReviewForAdvisory(env, { settings, advisory, repoFullName, pr, author, confirmedContributor }); + const gateEvaluation = gateEnabled ? evaluateGateCheck(advisory, gateCheckPolicy(settings, readiness.total, confirmedContributor)) : undefined; if (gateEnabled) { const gateCheckResult = await createOrUpdateGateCheckRun( @@ -1050,7 +1127,7 @@ async function maybePublishPrPublicSurface( // Maintainer review-content overrides from `.gittensory.yml` (footer text, row toggles, intro note). // Cached, so this is a DB read after the settings resolution already loaded the manifest. const reviewConfig = (await loadRepoFocusManifest(env, repoFullName)).review; - const commentArgs = { repo, pr, profile, detection, queueHealth, collisions, preflight, settings, gate: gateEvaluation, review: reviewConfig }; + const commentArgs = { repo, pr, profile, detection, queueHealth, collisions, preflight, settings, gate: gateEvaluation, review: reviewConfig, aiReview }; const deterministicBody = buildPublicPrIntelligenceComment(commentArgs); try { await createOrUpdatePrIntelligenceComment(env, installationId, repoFullName, pr.number, deterministicBody); diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index fac25550f8..e64962a1bf 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -19,6 +19,9 @@ export type GateCheckPolicy = { duplicatePrGateMode?: GateRuleMode | undefined; qualityGateMode?: GateRuleMode | undefined; qualityGateMinScore?: number | null | undefined; + /** When `block`, a dual-model AI consensus defect (`ai_consensus_defect` finding) becomes a hard + * blocker. Defaults to advisory — AI never blocks unless the maintainer opts in. */ + aiReviewGateMode?: GateRuleMode | undefined; readinessScore?: number | null | undefined; /** ONLY confirmed gittensor contributors can be hard-blocked. When explicitly `false`, the gate is * forced to a neutral (non-blocking) conclusion regardless of blockers — gittensory must never block @@ -558,6 +561,10 @@ function isConfiguredGateBlocker(code: string, policy: GateCheckPolicy): boolean // repo explicitly opts in with linkedIssueGateMode: "block". Duplicates still default to blocking. if (code === "missing_linked_issue") return gateMode(policy.linkedIssueGateMode ?? "advisory") === "block"; if (code === "duplicate_pr_risk") return gateMode(policy.duplicatePrGateMode ?? "block") === "block"; + // A dual-model AI consensus defect blocks ONLY when the maintainer opted into aiReview: block. It is the + // most conservative AI signal (two independent models, high confidence) but still confirmed-contributor + // gated by evaluateGateCheck, and advisory by default. + if (code === "ai_consensus_defect") return gateMode(policy.aiReviewGateMode ?? "advisory") === "block"; return false; } diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts new file mode 100644 index 0000000000..53a25af62e --- /dev/null +++ b/src/services/ai-review.ts @@ -0,0 +1,361 @@ +// Gittensory AI maintainer review (the `aiReview` capability). +// +// Two layers, both opt-in and both fail-safe (no AI / errors / over-budget / unsafe output → no public +// text and no gate finding; gittensory NEVER blocks because the model spoke): +// +// • Advisory notes — a concise maintainer-style write-up (assessment + suggestions + risks). When the +// repo has BYOK configured, the maintainer's own frontier model (Anthropic/OpenAI) writes it; +// otherwise free Cloudflare Workers AI does. Advisory only — never blocks. +// • Consensus defect — a conservative gate signal. The free Workers-AI model PAIR each independently +// reviews the diff; a defect is reported ONLY when BOTH models flag a high-confidence critical defect +// (bug / security / data-loss / build break). BYOK never changes this path, so it never changes who +// can be blocked. The resulting finding is honored by the gate only in `block` mode AND only for +// confirmed Gittensor contributors (the gate enforces that downstream). +// +// Every public string (notes + defect title/detail) is forced through `sanitizePublicComment`; anything +// that trips the public/private boundary is dropped, not published. Every model call is metered against +// the shared daily neuron budget and audited via `recordAiUsageEvent`. +import { recordAiUsageEvent, sumAiEstimatedNeuronsSince } from "../db/repositories"; +import { sanitizePublicComment } from "../queue-intelligence"; + +/** + * The best free Workers-AI model pair for review accuracy — two different families for independence, + * both probe-verified in reviewbot to emit clean JSON. The consensus blocker always uses this pair. + */ +export const BEST_REVIEW_MODELS: readonly [string, string] = ["@cf/openai/gpt-oss-120b", "@cf/nvidia/nemotron-3-120b-a12b"]; + +/** Reliable per-slot fallbacks (non-reasoning, clean JSON) so a slot never comes back empty. */ +export const RELIABLE_FALLBACK_MODELS: readonly [string, string] = [ + "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + "@cf/mistralai/mistral-small-3.1-24b-instruct", +]; + +/** Default consensus confidence floor: BOTH models must be at/above this to report a defect. */ +export const AI_CONSENSUS_FLOOR = 0.9; + +const REVIEW_SYSTEM_PROMPT = [ + "You are a senior open-source maintainer reviewing a single pull request diff.", + "Be concise, concrete, and fair. Judge only the diff and the context provided.", + "Report a critical defect ONLY when you are highly confident the change introduces a real bug, a", + "security hole, data loss, or a build break — NOT for style, nits, naming, or merely-missing tests.", + "Never mention rewards, rankings, payouts, wallets, hotkeys, coldkeys, trust scores, scoreability,", + "reviewability, or farming.", + 'Respond with ONLY a JSON object of this exact shape (no prose, no code fence):', + '{"assessment": string, "suggestions": string[], "risks": string[],', + ' "criticalDefect": {"present": boolean, "confidence": number, "title": string, "detail": string}}', +].join(" "); + +/** A maintainer's BYOK provider credential, decrypted at call time. Never logged, never returned. */ +export type AiReviewProviderKey = { + provider: "anthropic" | "openai"; + key: string; + /** Optional model override; falls back to a conservative stable default per provider. */ + model?: string | null | undefined; +}; + +export type GittensoryAiReviewInput = { + repoFullName: string; + prNumber: number; + title: string; + body?: string | null | undefined; + /** A bounded unified-diff-ish string built by the caller (filenames + patches). */ + diff: string; + actor?: string | null | undefined; + /** Effective `aiReviewMode`. `block` additionally runs the consensus-defect pass. */ + mode: "advisory" | "block"; + /** Present only when the repo has BYOK on AND a key configured; drives the advisory write-up. */ + providerKey?: AiReviewProviderKey | null | undefined; +}; + +/** A consensus critical defect, already public-safe, ready to become a gate blocker finding. */ +export type AiConsensusDefect = { title: string; detail: string; confidence: number }; + +export type GittensoryAiReviewResult = + | { status: "disabled"; reason: string } + | { status: "unavailable"; reason: string } + | { status: "quota_exceeded"; estimatedNeurons: number; remainingBudget: number } + | { status: "ok"; advisoryNotes: string | null; consensusDefect: AiConsensusDefect | null; estimatedNeurons: number }; + +type ModelReview = { + assessment: string; + suggestions: string[]; + risks: string[]; + criticalDefect: { present: boolean; confidence: number; title: string; detail: string }; +}; + +type AiRunner = { run?: (model: string, options: Record) => Promise }; + +function isEnabled(value: string | undefined): boolean { + return /^(1|true|yes|on)$/i.test(value ?? ""); +} + +function clampNumber(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(max, Math.max(min, Math.floor(value))); +} + +function utcDayStartIso(): string { + const now = new Date(); + return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())).toISOString(); +} + +function estimateNeurons(promptChars: number, maxOutputTokens: number, calls: number): number { + const inputTokens = Math.ceil(promptChars / 4); + return Math.max(1, Math.ceil((inputTokens + maxOutputTokens) * 0.035) * Math.max(1, calls)); +} + +/** Returns the text unchanged if it is public-safe, otherwise null (drop — never publish). */ +export function toPublicSafe(text: string | null | undefined): string | null { + const trimmed = (text ?? "").trim(); + if (!trimmed) return null; + try { + return sanitizePublicComment(trimmed); + } catch { + return null; + } +} + +/** Coerce the varied Workers-AI / provider response envelopes into a scannable string. */ +export function coerceAiText(result: unknown): string { + if (typeof result === "string") return result; + if (result && typeof result === "object") { + const obj = result as Record; + const response = obj.response; + if (typeof response === "string" && response.trim()) return response; + if (response && typeof response === "object") return JSON.stringify(response); + const choices = obj.choices; + if (Array.isArray(choices) && choices.length > 0) { + const first = choices[0] as { message?: { content?: unknown }; text?: unknown }; + const content = first?.message?.content ?? first?.text; + if (typeof content === "string" && content.trim()) return content; + } + // Anthropic Messages: { content: [{ type: "text", text }] } + const content = obj.content; + if (Array.isArray(content) && content.length > 0) { + const parts = content + .map((part) => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string" ? (part as { text: string }).text : "")) + .filter(Boolean); + if (parts.length > 0) return parts.join("\n"); + } + if (typeof obj.output_text === "string" && obj.output_text.trim()) return obj.output_text; + } + return ""; +} + +/** Parse a model's JSON review into a normalized {@link ModelReview}, or null when unparseable. */ +export function parseModelReview(text: string): ModelReview | null { + const match = text.replace(/^```(?:json)?\s*/i, "").replace(/```$/i, "").match(/\{[\s\S]*\}/); + if (!match) return null; + try { + const obj = JSON.parse(match[0]) as Record; + const toList = (value: unknown): string[] => + Array.isArray(value) ? value.filter((x): x is string => typeof x === "string").map((x) => x.trim()).filter(Boolean).slice(0, 6) : []; + const assessment = typeof obj.assessment === "string" ? obj.assessment.trim() : ""; + const defectRaw = obj.criticalDefect && typeof obj.criticalDefect === "object" ? (obj.criticalDefect as Record) : {}; + const present = defectRaw.present === true; + const confidence = typeof defectRaw.confidence === "number" ? Math.max(0, Math.min(1, defectRaw.confidence)) : 0; + if (!assessment && !present && !Array.isArray(obj.suggestions)) return null; + return { + assessment, + suggestions: toList(obj.suggestions), + risks: toList(obj.risks), + criticalDefect: { + present, + confidence, + title: typeof defectRaw.title === "string" ? defectRaw.title.trim().slice(0, 140) : "", + detail: typeof defectRaw.detail === "string" ? defectRaw.detail.trim().slice(0, 400) : "", + }, + }; + } catch { + return null; + } +} + +function buildUserPrompt(input: GittensoryAiReviewInput): string { + return [ + `Repository: ${input.repoFullName}`, + `Pull request #${input.prNumber}: ${input.title}`, + input.body ? `Description:\n${input.body.slice(0, 2000)}` : "Description: (none)", + "", + "Unified diff (truncated if large):", + input.diff.slice(0, 60000), + ].join("\n"); +} + +/** One Workers-AI opinion with a per-slot reliable fallback and a 3× retry on the primary. */ +async function runWorkersOpinion(env: Env, primary: string, fallback: string, system: string, user: string, maxTokens: number): Promise { + const ai = env.AI as unknown as AiRunner | undefined; + if (!ai || typeof ai.run !== "function") return null; + for (const model of fallback && fallback !== primary ? [primary, fallback] : [primary]) { + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + const result = await ai.run(model, { + max_tokens: maxTokens, + temperature: 0, + messages: [ + { role: "system", content: system }, + { role: "user", content: user }, + ], + }); + const parsed = parseModelReview(coerceAiText(result)); + if (parsed) return parsed; + } catch { + /* retry / fall through to fallback */ + } + } + } + return null; +} + +const PROVIDER_DEFAULT_MODEL: Record = { + anthropic: "claude-3-5-sonnet-latest", + openai: "gpt-4o", +}; + +/** Run the maintainer's BYOK frontier model for the advisory write-up. Never throws; null on any error. */ +async function runProviderReview(providerKey: AiReviewProviderKey, system: string, user: string, maxTokens: number): Promise { + const model = providerKey.model || PROVIDER_DEFAULT_MODEL[providerKey.provider]; + try { + let response: Response; + if (providerKey.provider === "anthropic") { + response = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": providerKey.key, "anthropic-version": "2023-06-01" }, + body: JSON.stringify({ model, max_tokens: maxTokens, system, messages: [{ role: "user", content: user }] }), + }); + } else { + response = await fetch("https://api.openai.com/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${providerKey.key}` }, + body: JSON.stringify({ + model, + max_tokens: maxTokens, + messages: [ + { role: "system", content: system }, + { role: "user", content: user }, + ], + }), + }); + } + if (!response.ok) return null; + return parseModelReview(coerceAiText(await response.json())); + } catch { + return null; + } +} + +/** Compose a public-safe markdown advisory blurb from one or two model reviews. Null if nothing safe. */ +export function composeAdvisoryNotes(reviews: ModelReview[]): string | null { + const assessments = reviews.map((r) => r.assessment).filter(Boolean); + const suggestions = [...new Set(reviews.flatMap((r) => r.suggestions))].slice(0, 5); + const risks = [...new Set(reviews.flatMap((r) => r.risks))].slice(0, 4); + const assessment = toPublicSafe(assessments[0] ?? ""); + const safeSuggestions = suggestions.map((s) => toPublicSafe(s)).filter((s): s is string => Boolean(s)); + const safeRisks = risks.map((s) => toPublicSafe(s)).filter((s): s is string => Boolean(s)); + if (!assessment && safeSuggestions.length === 0 && safeRisks.length === 0) return null; + const lines: string[] = []; + if (assessment) lines.push(assessment, ""); + if (safeSuggestions.length > 0) { + lines.push("**Suggestions**"); + lines.push(...safeSuggestions.map((s) => `- ${s}`)); + lines.push(""); + } + if (safeRisks.length > 0) { + lines.push("**Risks**"); + lines.push(...safeRisks.map((s) => `- ${s}`)); + } + // Reaching here means at least one section was pushed (the all-empty case returned null above). + return lines.join("\n").trim(); +} + +/** True iff BOTH reviews independently report a critical defect at/above the floor. */ +export function consensusDefectOf(a: ModelReview, b: ModelReview, floor: number): AiConsensusDefect | null { + const both = a.criticalDefect.present && b.criticalDefect.present && a.criticalDefect.confidence >= floor && b.criticalDefect.confidence >= floor; + if (!both) return null; + const title = toPublicSafe(a.criticalDefect.title || b.criticalDefect.title || "AI reviewers agree on a likely critical defect"); + const detail = toPublicSafe(a.criticalDefect.detail || b.criticalDefect.detail); + if (!title) return null; // unsafe title → drop the block entirely (fail-safe) + return { title, detail: detail ?? "Both AI reviewers independently flagged a high-confidence critical defect in this change.", confidence: Math.min(a.criticalDefect.confidence, b.criticalDefect.confidence) }; +} + +/** + * Run the AI maintainer review. Returns advisory notes (always, when AI is on) and — in `block` mode — + * a consensus defect when the free Workers-AI pair agrees with high confidence. Fail-safe on every error + * path: no notes, no defect, never a thrown error reaching the webhook. + */ +export async function runGittensoryAiReview(env: Env, input: GittensoryAiReviewInput): Promise { + if (!isEnabled(env.AI_SUMMARIES_ENABLED)) return { status: "disabled", reason: "AI summaries are disabled." }; + if (!isEnabled(env.AI_PUBLIC_COMMENTS_ENABLED)) return { status: "disabled", reason: "Public AI comments are disabled." }; + if (!env.AI) return { status: "unavailable", reason: "Workers AI binding is not configured." }; + + const maxTokens = clampNumber(Number(env.AI_MAX_OUTPUT_TOKENS || 256), 256, 1024); + const user = buildUserPrompt(input); + // block mode = advisory pass + consensus pass (2 models each, minus 1 when BYOK writes the advisory). + const calls = input.mode === "block" ? 3 : input.providerKey ? 1 : 2; + const estimatedNeurons = estimateNeurons(REVIEW_SYSTEM_PROMPT.length + user.length, maxTokens, calls); + const budget = clampNumber(Number(env.AI_DAILY_NEURON_BUDGET || 10000), 0, 1_000_000); + const used = await sumAiEstimatedNeuronsSince(env, utcDayStartIso()); + const remainingBudget = Math.max(0, budget - used); + if (estimatedNeurons > remainingBudget) { + await record(env, input, "quota_exceeded", 0, `estimated ${estimatedNeurons} neurons exceeds remaining ${remainingBudget}`); + return { status: "quota_exceeded", estimatedNeurons, remainingBudget }; + } + + // Advisory write-up: BYOK frontier model if configured, else the free Workers-AI primary (with fallback). + const advisoryReview = input.providerKey + ? await runProviderReview(input.providerKey, REVIEW_SYSTEM_PROMPT, user, maxTokens) + : await runWorkersOpinion(env, BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0], REVIEW_SYSTEM_PROMPT, user, maxTokens); + + let consensusDefect: AiConsensusDefect | null = null; + let secondReview: ModelReview | null = null; + if (input.mode === "block") { + // Consensus blocker ALWAYS uses the free Workers-AI pair (provider-independent, never BYOK). + const [a, b] = await Promise.all([ + input.providerKey ? runWorkersOpinion(env, BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0], REVIEW_SYSTEM_PROMPT, user, maxTokens) : Promise.resolve(advisoryReview), + runWorkersOpinion(env, BEST_REVIEW_MODELS[1], RELIABLE_FALLBACK_MODELS[1], REVIEW_SYSTEM_PROMPT, user, maxTokens), + ]); + secondReview = b; + if (a && b) consensusDefect = consensusDefectOf(a, b, AI_CONSENSUS_FLOOR); + } + + const reviewsForNotes = [advisoryReview, secondReview].filter((r): r is ModelReview => Boolean(r)); + const advisoryNotes = reviewsForNotes.length > 0 ? composeAdvisoryNotes(reviewsForNotes) : null; + + await record(env, input, "ok", estimatedNeurons, consensusDefect ? "consensus defect" : advisoryNotes ? "advisory notes" : "no usable output", { + mode: input.mode, + byok: Boolean(input.providerKey), + consensus: Boolean(consensusDefect), + }); + return { status: "ok", advisoryNotes, consensusDefect, estimatedNeurons }; +} + +async function record( + env: Env, + input: GittensoryAiReviewInput, + status: string, + estimatedNeurons: number, + detail: string, + metadata?: Record, +): Promise { + // NEVER include provider key material in usage/audit metadata. + await recordAiUsageEvent(env, { + feature: "ai_review_pr", + actor: input.actor ?? null, + route: "github_app.ai_review", + model: input.providerKey ? `byok:${input.providerKey.provider}` : BEST_REVIEW_MODELS.join("+"), + status, + estimatedNeurons, + detail, + metadata: { repoFullName: input.repoFullName, pullNumber: input.prNumber, ...(metadata ?? {}) }, + }); +} + +export const __aiReviewInternals = { + parseModelReview, + coerceAiText, + composeAdvisoryNotes, + consensusDefectOf, + toPublicSafe, + estimateNeurons, + runWorkersOpinion, +}; diff --git a/src/signals/engine.ts b/src/signals/engine.ts index f25eb81afd..62d0368564 100644 --- a/src/signals/engine.ts +++ b/src/signals/engine.ts @@ -3859,6 +3859,8 @@ export function buildPublicPrIntelligenceComment(args: { settings: RepositorySettings; gate?: PublicPrPanelGateEvaluation | undefined; review?: FocusManifestReviewConfig | undefined; + /** Optional AI maintainer-review notes (already public-safe). Rendered as an advisory section. */ + aiReview?: { notes: string } | undefined; }): string { const publicFindings = args.preflight.findings .filter((finding) => finding.severity !== "critical") @@ -4016,6 +4018,24 @@ export function buildPublicPrIntelligenceComment(args: { ...(nextSteps.length > 0 ? [...new Set(nextSteps)].map((step) => `- ${step}`) : ["- Keep the PR focused and include validation evidence before maintainer review."]), "", "", + // Optional AI maintainer review (advisory; public-safe text built upstream). The deterministic + // signals above remain authoritative — this is a second opinion, not an endorsement. + ...(args.aiReview + ? [ + "", + "
", + "Gittensory AI review (advisory)", + "", + "_Generated from public PR metadata and the diff. Advisory only; deterministic signals remain authoritative._", + "", + // Notes are already public-safe (built via toPublicSafe upstream). Escape angle brackets so a + // stray tag (e.g.
or an HTML comment marker) cannot break the panel structure, while + // preserving the markdown bullet/line layout that sanitizePanelText would otherwise flatten. + args.aiReview.notes.replace(/[<>]/g, (char) => (char === "<" ? "<" : ">")).slice(0, 4000), + "", + "", + ] + : []), "", `- [ ] ${PR_PANEL_RETRIGGER_MARKER} Re-run Gittensory review`, "", diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts new file mode 100644 index 0000000000..d9db737fb7 --- /dev/null +++ b/test/unit/ai-review-advisory.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { buildAiReviewDiff, runAiReviewForAdvisory } from "../../src/queue/processors"; +import type { Advisory, PullRequestFileRecord, RepositorySettings } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +function fileRecord(over: Partial & { path: string }): PullRequestFileRecord { + return { repoFullName: "acme/widgets", pullNumber: 3, status: "modified", additions: 1, deletions: 0, changes: 1, payload: {}, ...over }; +} + +describe("buildAiReviewDiff", () => { + it("includes patches and headers, omits the patch when absent, and truncates oversized diffs", () => { + const diff = buildAiReviewDiff([ + fileRecord({ path: "src/a.ts", status: "modified", payload: { patch: "@@\n+const x = 1;" } }), + fileRecord({ path: "src/b.ts", status: undefined, payload: {} }), + ]); + expect(diff).toContain("### src/a.ts (modified) +1/-0"); + expect(diff).toContain("+const x = 1;"); + expect(diff).toContain("### src/b.ts +1/-0"); // no status, no patch + expect(buildAiReviewDiff([])).toBe(""); + + const huge = buildAiReviewDiff([fileRecord({ path: "src/big.ts", payload: { patch: "x".repeat(70000) } }), fileRecord({ path: "src/next.ts" })]); + expect(huge).toContain("diff truncated"); + }); +}); + +function advisory(over: Partial = {}): Advisory { + return { + id: "adv-1", + targetType: "pull_request", + targetKey: "acme/widgets#3", + repoFullName: "acme/widgets", + pullNumber: 3, + headSha: "sha3", + conclusion: "neutral", + severity: "info", + title: "Gittensory advisory available", + summary: "ok", + findings: [], + generatedAt: "2026-06-13T00:00:00.000Z", + ...over, + }; +} + +const pr = { number: 3, title: "Add helper", body: "Adds a helper." }; + +function defectJson() { + return JSON.stringify({ assessment: "Likely crash.", suggestions: ["Guard null."], risks: ["Null deref."], criticalDefect: { present: true, confidence: 0.97, title: "Null deref", detail: "Dereferences null." } }); +} +function notesOnlyJson() { + return JSON.stringify({ assessment: "Looks fine.", suggestions: ["Add a test."], risks: [], criticalDefect: { present: false, confidence: 0, title: "", detail: "" } }); +} + +function aiEnv(run: () => Promise, flags = true) { + return createTestEnv({ + AI: { run } as unknown as Ai, + ...(flags ? { AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" } : {}), + AI_DAILY_NEURON_BUDGET: "100000", + }); +} + +describe("runAiReviewForAdvisory", () => { + it("no-ops when aiReviewMode is off", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: defectJson() })), { + settings: { aiReviewMode: "off" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toBeUndefined(); + expect(adv.findings).toEqual([]); + }); + + it("no-ops for a non-confirmed contributor and when there is no head SHA", async () => { + const env = aiEnv(async () => ({ response: defectJson() })); + const base = { settings: { aiReviewMode: "block" } as RepositorySettings, repoFullName: "acme/widgets", pr, author: "alice" }; + expect(await runAiReviewForAdvisory(env, { ...base, advisory: advisory(), confirmedContributor: false })).toBeUndefined(); + const noSha = advisory(); + delete (noSha as Partial).headSha; + expect(await runAiReviewForAdvisory(env, { ...base, advisory: noSha, confirmedContributor: true })).toBeUndefined(); + }); + + it("appends an ai_consensus_defect finding in block mode when the models agree", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: defectJson() })), { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(adv.findings.map((f) => f.code)).toEqual(["ai_consensus_defect"]); + expect(adv.findings[0]?.title).toContain("Null deref"); + expect(result?.notes).toContain("Likely crash."); + }); + + it("returns advisory notes without a finding in advisory mode", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: notesOnlyJson() })), { + settings: { aiReviewMode: "advisory" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(adv.findings).toEqual([]); + expect(result?.notes).toContain("Add a test."); + }); + + it("returns undefined (no notes, no finding) when AI is disabled", async () => { + const adv = advisory(); + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: defectJson() }), false), { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toBeUndefined(); + expect(adv.findings).toEqual([]); + }); + + it("returns undefined when the model produces no parseable notes", async () => { + const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "not json" })), { + settings: { aiReviewMode: "advisory" } as RepositorySettings, + advisory: advisory(), + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toBeUndefined(); + }); + + it("is fail-safe: a thrown error (e.g. broken DB) yields no finding and no notes", async () => { + const adv = advisory(); + const env = aiEnv(async () => ({ response: defectJson() })); + const result = await runAiReviewForAdvisory({ ...env, DB: undefined } as unknown as Env, { + settings: { aiReviewMode: "block" } as RepositorySettings, + advisory: adv, + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result).toBeUndefined(); + expect(adv.findings).toEqual([]); + }); +}); diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts new file mode 100644 index 0000000000..92e70cf902 --- /dev/null +++ b/test/unit/ai-review.test.ts @@ -0,0 +1,269 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + __aiReviewInternals, + AI_CONSENSUS_FLOOR, + BEST_REVIEW_MODELS, + runGittensoryAiReview, + type GittensoryAiReviewInput, +} from "../../src/services/ai-review"; +import { createTestEnv } from "../helpers/d1"; + +const { parseModelReview, coerceAiText, composeAdvisoryNotes, consensusDefectOf, toPublicSafe, runWorkersOpinion } = __aiReviewInternals; + +function reviewJson(over: Partial<{ assessment: string; suggestions: string[]; risks: string[]; present: boolean; confidence: number; title: string; detail: string }> = {}): string { + return JSON.stringify({ + assessment: over.assessment ?? "The change looks reasonable and focused.", + suggestions: over.suggestions ?? ["Add a unit test for the new branch."], + risks: over.risks ?? ["Edge case on empty input is untested."], + criticalDefect: { present: over.present ?? false, confidence: over.confidence ?? 0, title: over.title ?? "", detail: over.detail ?? "" }, + }); +} + +const baseInput: GittensoryAiReviewInput = { + repoFullName: "acme/widgets", + prNumber: 7, + title: "Fix null deref", + body: "Closes #1", + diff: "### src/a.ts (modified) +3/-1\n@@\n+const x = 1;", + actor: "alice", + mode: "advisory", +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("runGittensoryAiReview gating", () => { + it("is disabled until both AI flags are on", async () => { + const run = vi.fn(); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true" }); + await expect(runGittensoryAiReview(env, baseInput)).resolves.toMatchObject({ status: "disabled" }); + expect(run).not.toHaveBeenCalled(); + }); + + it("reports unavailable when the Workers AI binding is missing", async () => { + const env = createTestEnv({ AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await expect(runGittensoryAiReview(env, baseInput)).resolves.toMatchObject({ status: "unavailable" }); + }); + + it("enforces the shared daily neuron budget before calling the model", async () => { + const run = vi.fn(); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "1" }); + await expect(runGittensoryAiReview(env, baseInput)).resolves.toMatchObject({ status: "quota_exceeded" }); + expect(run).not.toHaveBeenCalled(); + }); +}); + +describe("runGittensoryAiReview advisory mode", () => { + it("produces public-safe advisory notes from one Workers-AI opinion and no defect", async () => { + const run = vi.fn(async (_model: string) => ({ response: reviewJson() })); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, baseInput); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(result.consensusDefect).toBeNull(); + expect(result.advisoryNotes).toContain("Suggestions"); + expect(result.advisoryNotes).toContain("Add a unit test"); + // Advisory mode runs a single opinion (primary model). + expect(run).toHaveBeenCalledTimes(1); + expect(run.mock.calls[0]?.[0]).toBe(BEST_REVIEW_MODELS[0]); + }); +}); + +describe("runGittensoryAiReview block mode (consensus)", () => { + function envWith(run: (model: string) => Promise) { + return createTestEnv({ AI: { run: vi.fn(run) } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + } + + it("reports a consensus defect only when BOTH models agree at/above the floor", async () => { + const env = envWith(async () => ({ response: reviewJson({ present: true, confidence: 0.95, title: "Unhandled null", detail: "Crashes on empty list." }) })); + const result = await runGittensoryAiReview(env, { ...baseInput, mode: "block" }); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(result.consensusDefect).not.toBeNull(); + expect(result.consensusDefect?.title).toContain("Unhandled null"); + }); + + it("does NOT report a defect when only one model flags it", async () => { + const env = envWith(async (model) => + model === BEST_REVIEW_MODELS[1] + ? { response: reviewJson({ present: false }) } + : { response: reviewJson({ present: true, confidence: 0.99, title: "Race", detail: "Concurrent write." }) }, + ); + const result = await runGittensoryAiReview(env, { ...baseInput, mode: "block" }); + expect(result.status === "ok" && result.consensusDefect).toBeNull(); + }); + + it("does NOT report a defect when both agree but below the confidence floor", async () => { + const env = envWith(async () => ({ response: reviewJson({ present: true, confidence: 0.6, title: "Maybe bug", detail: "Unsure." }) })); + const result = await runGittensoryAiReview(env, { ...baseInput, mode: "block" }); + expect(result.status === "ok" && result.consensusDefect).toBeNull(); + }); + + it("does NOT report a defect when one model's verdict is unparseable (null opinion)", async () => { + // Only the first slot's primary parses; the second slot's primary AND its reliable fallback fail. + const env = envWith(async (model) => + model === BEST_REVIEW_MODELS[0] ? { response: reviewJson({ present: true, confidence: 0.99, title: "Null deref", detail: "boom" }) } : { response: "garbage" }, + ); + const result = await runGittensoryAiReview(env, { ...baseInput, mode: "block", actor: undefined }); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(result.consensusDefect).toBeNull(); + expect(result.advisoryNotes).not.toBeNull(); // notes still come from the one parseable opinion + }); + + it("block mode with BYOK: provider writes the advisory, the free Workers-AI pair drives consensus", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ content: [{ type: "text", text: reviewJson({ assessment: "Frontier advisory." }) }] }), { status: 200 }))); + const run = vi.fn(async (_model: string) => ({ response: reviewJson({ present: true, confidence: 0.96, title: "Off-by-one", detail: "Loop bound." }) })); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, { ...baseInput, mode: "block", providerKey: { provider: "anthropic", key: "sk-ant" } }); + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(result.consensusDefect?.title).toContain("Off-by-one"); // consensus from Workers AI, not the provider + expect(result.advisoryNotes).toContain("Frontier advisory."); // advisory from BYOK provider + expect(run).toHaveBeenCalledTimes(2); // both consensus opinions via Workers AI + }); +}); + +describe("BYOK provider dispatch", () => { + it("uses the Anthropic API for the advisory write-up when a key is supplied", async () => { + const fetchMock = vi.fn(async (_url: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({ content: [{ type: "text", text: reviewJson({ assessment: "BYOK review." }) }] }), { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const run = vi.fn(); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, { ...baseInput, providerKey: { provider: "anthropic", key: "sk-ant-secret" } }); + expect(result.status === "ok" && result.advisoryNotes).toContain("BYOK review."); + expect(fetchMock.mock.calls[0]?.[0]).toBe("https://api.anthropic.com/v1/messages"); + expect(run).not.toHaveBeenCalled(); // advisory mode + BYOK → no Workers AI call + }); + + it("falls back to no notes when the provider returns a non-200", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("nope", { status: 401 }))); + const env = createTestEnv({ AI: { run: vi.fn() } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, { ...baseInput, providerKey: { provider: "openai", key: "sk-secret" } }); + expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + }); + + it("falls back to no notes when the provider fetch throws, and honors a model override", async () => { + const fetchMock = vi.fn(async (_url: RequestInfo | URL, _init?: RequestInit) => { + throw new Error("network down"); + }); + vi.stubGlobal("fetch", fetchMock); + const env = createTestEnv({ AI: { run: vi.fn() } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, { ...baseInput, providerKey: { provider: "anthropic", key: "sk-ant", model: "claude-custom" } }); + expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1] && (fetchMock.mock.calls[0][1] as RequestInit).body)).model).toBe("claude-custom"); + }); +}); + +describe("Workers AI fallback + degraded output", () => { + it("tries the per-slot fallback model then returns no notes when every opinion is unparseable", async () => { + const run = vi.fn(async (_model: string) => ({ response: "this is not json at all" })); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, baseInput); + expect(result.status === "ok" && result.advisoryNotes).toBeNull(); + // primary 3× + fallback 3× retries, all unparseable. + expect(run).toHaveBeenCalledTimes(6); + }); +}); + +describe("pure helpers", () => { + it("toPublicSafe drops forbidden public text and keeps safe text", () => { + expect(toPublicSafe("This change is solid.")).toBe("This change is solid."); + expect(toPublicSafe("Boost your reward payout")).toBeNull(); + expect(toPublicSafe("")).toBeNull(); + }); + + it("coerceAiText handles string, {response}, OpenAI choices, Anthropic content, and output_text shapes", () => { + expect(coerceAiText("raw")).toBe("raw"); + expect(coerceAiText({ response: "r" })).toBe("r"); + expect(coerceAiText({ choices: [{ message: { content: "c" } }] })).toBe("c"); + expect(coerceAiText({ content: [{ type: "text", text: "a" }] })).toBe("a"); + expect(coerceAiText({ content: [] })).toBe(""); // empty content array + expect(coerceAiText({ content: [{ type: "image" }], output_text: "fallback" })).toBe("fallback"); // non-text parts → fall through + expect(coerceAiText({ output_text: "o" })).toBe("o"); + expect(coerceAiText(42)).toBe(""); + }); + + it("parseModelReview returns null on junk, on brace-but-invalid JSON, on empty objects, and clamps confidence", () => { + expect(parseModelReview("not json")).toBeNull(); + expect(parseModelReview("{ not: valid json }")).toBeNull(); // matches the brace regex but JSON.parse throws + expect(parseModelReview('{"foo":1}')).toBeNull(); // no assessment, no defect, no suggestions + const parsed = parseModelReview(reviewJson({ present: true, confidence: 5, title: "X", detail: "Y" })); + expect(parsed?.criticalDefect.confidence).toBe(1); + }); + + it("parseModelReview coerces non-string/non-array fields to safe defaults", () => { + const parsed = parseModelReview('{"assessment":"ok","suggestions":"not-an-array","risks":7,"criticalDefect":{"present":true,"confidence":0.9,"title":5,"detail":null}}'); + expect(parsed).not.toBeNull(); + expect(parsed?.suggestions).toEqual([]); // non-array → [] + expect(parsed?.risks).toEqual([]); + expect(parsed?.criticalDefect.title).toBe(""); // non-string → "" + expect(parsed?.criticalDefect.detail).toBe(""); + }); + + it("consensusDefectOf requires both present and at/above the floor and drops unsafe titles", () => { + const defect = (present: boolean, confidence: number, title = "Null deref", detail = "boom") => ({ assessment: "", suggestions: [], risks: [], criticalDefect: { present, confidence, title, detail } }); + expect(consensusDefectOf(defect(true, 0.95), defect(true, 0.95), AI_CONSENSUS_FLOOR)).not.toBeNull(); + expect(consensusDefectOf(defect(true, 0.8), defect(true, 0.95), AI_CONSENSUS_FLOOR)).toBeNull(); + expect(consensusDefectOf(defect(false, 0.95), defect(true, 0.95), AI_CONSENSUS_FLOOR)).toBeNull(); // one not present + expect(consensusDefectOf(defect(true, 0.95, "Boost your reward payout"), defect(true, 0.95, "Boost your reward payout"), AI_CONSENSUS_FLOOR)).toBeNull(); + }); + + it("consensusDefectOf falls back to b's title and a default detail when a is blank", () => { + const a = { assessment: "", suggestions: [], risks: [], criticalDefect: { present: true, confidence: 0.95, title: "", detail: "" } }; + const b = { assessment: "", suggestions: [], risks: [], criticalDefect: { present: true, confidence: 0.93, title: "Race condition", detail: "" } }; + const out = consensusDefectOf(a, b, AI_CONSENSUS_FLOOR); + expect(out?.title).toBe("Race condition"); + expect(out?.detail).toMatch(/independently flagged/); + // both titles blank → default title string is used + const blank = { ...a, criticalDefect: { ...a.criticalDefect } }; + expect(consensusDefectOf(blank, { ...blank, criticalDefect: { ...blank.criticalDefect } }, AI_CONSENSUS_FLOOR)?.title).toContain("AI reviewers agree"); + }); + + it("runWorkersOpinion returns null without a binding and handles a single-model (no distinct fallback) list", async () => { + expect(await runWorkersOpinion(createTestEnv({}), "m", "f", "sys", "user", 256)).toBeNull(); + const run = vi.fn(async (_model: string) => ({ response: reviewJson() })); + const env = createTestEnv({ AI: { run } as unknown as Ai }); + // fallback === primary exercises the single-element model list branch. + const parsed = await runWorkersOpinion(env, "@cf/x/model", "@cf/x/model", "sys", "user", 256); + expect(parsed?.assessment).toContain("reasonable"); + expect(run).toHaveBeenCalledTimes(1); + }); + + it("applies the default daily neuron budget when none is configured", async () => { + const run = vi.fn(async (_model: string) => ({ response: reviewJson() })); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + const result = await runGittensoryAiReview(env, baseInput); + expect(result.status).toBe("ok"); + }); + + it("composeAdvisoryNotes returns null when nothing is public-safe", () => { + expect(composeAdvisoryNotes([{ assessment: "reward payout farming", suggestions: ["payout"], risks: ["reward"], criticalDefect: { present: false, confidence: 0, title: "", detail: "" } }])).toBeNull(); + }); + + it("composeAdvisoryNotes renders only the sections that have public-safe content", () => { + const review = (over: Partial<{ assessment: string; suggestions: string[]; risks: string[] }>) => ({ assessment: over.assessment ?? "", suggestions: over.suggestions ?? [], risks: over.risks ?? [], criticalDefect: { present: false, confidence: 0, title: "", detail: "" } }); + const assessmentOnly = composeAdvisoryNotes([review({ assessment: "Looks good." })]); + expect(assessmentOnly).toBe("Looks good."); + const suggestionsOnly = composeAdvisoryNotes([review({ suggestions: ["Add a test."] })]); + expect(suggestionsOnly).toContain("**Suggestions**"); + expect(suggestionsOnly).not.toContain("**Risks**"); + const risksOnly = composeAdvisoryNotes([review({ risks: ["Edge case."] })]); + expect(risksOnly).toContain("**Risks**"); + expect(risksOnly).not.toContain("**Suggestions**"); + }); + + it("runGittensoryAiReview is disabled when neither flag is set", async () => { + const env = createTestEnv({ AI: { run: vi.fn() } as unknown as Ai }); + await expect(runGittensoryAiReview(env, baseInput)).resolves.toMatchObject({ status: "disabled", reason: "AI summaries are disabled." }); + }); + + it("handles a review input with no PR body", async () => { + const run = vi.fn(async (_model: string, _options: { messages: Array<{ content: string }> }) => ({ response: reviewJson() })); + const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true", AI_DAILY_NEURON_BUDGET: "100000" }); + const result = await runGittensoryAiReview(env, { ...baseInput, body: undefined }); + expect(result.status).toBe("ok"); + expect(String(run.mock.calls[0]?.[1] && (run.mock.calls[0][1] as { messages: Array<{ content: string }> }).messages[1]?.content)).toContain("Description: (none)"); + }); +}); diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 2f6aae0777..866c2843e2 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -83,3 +83,28 @@ describe(".gittensory.yml settings override (resolveEffectiveSettings)", () => { expect(nonConfirmed.blockers).toEqual([]); }); }); + +describe("AI consensus defect gate blocker", () => { + function aiDefectAdvisory(): Advisory { + return { ...missingIssueAdvisory(), findings: [{ code: "ai_consensus_defect", title: "AI reviewers agree on a likely critical defect", severity: "critical", detail: "Both models flagged a null deref.", action: "Resolve it." }] }; + } + + it("is advisory by default — an AI consensus defect does NOT block when aiReview mode is off/advisory", () => { + expect(evaluateGateCheck(aiDefectAdvisory(), gateCheckPolicy(settings({ aiReviewMode: "off" }), null, true)).conclusion).toBe("success"); + expect(evaluateGateCheck(aiDefectAdvisory(), gateCheckPolicy(settings({ aiReviewMode: "advisory" }), null, true)).conclusion).toBe("success"); + }); + + it("blocks a confirmed contributor when the maintainer opts into aiReview: block (incl. via .gittensory.yml)", () => { + const blocked = evaluateGateCheck(aiDefectAdvisory(), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, true)); + expect(blocked.conclusion).toBe("failure"); + expect(blocked.blockers.map((f) => f.code)).toEqual(["ai_consensus_defect"]); + const eff = resolveEffectiveSettings(settings({ aiReviewMode: "off" }), parseFocusManifest({ gate: { aiReview: { mode: "block" } } })); + expect(evaluateGateCheck(aiDefectAdvisory(), gateCheckPolicy(eff, null, true)).conclusion).toBe("failure"); + }); + + it("never blocks a non-confirmed contributor even with aiReview: block", () => { + const result = evaluateGateCheck(aiDefectAdvisory(), gateCheckPolicy(settings({ aiReviewMode: "block" }), null, false)); + expect(result.conclusion).toBe("neutral"); + expect(result.blockers).toEqual([]); + }); +}); diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 37abcaa861..8356aee3ce 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -968,6 +968,82 @@ describe("queue processors", () => { expect(gatePatchBody.output?.title).toBe("Gittensory Gate: No linked issue detected"); }); + it("hard-blocks a confirmed contributor on a dual-model AI consensus defect when aiReview: block is opted in", async () => { + const defectJson = JSON.stringify({ + assessment: "Introduces a likely crash.", + suggestions: ["Guard the null case."], + risks: ["Unhandled null on empty input."], + criticalDefect: { present: true, confidence: 0.96, title: "Unhandled null dereference", detail: "The new branch dereferences a possibly-null value." }, + }); + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: defectJson }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await persistRegistrySnapshot( + env, + normalizeRegistryPayload( + { "JSONbored/gittensory": { emission_share: 0.01, issue_discovery_share: 0 } }, + { kind: "raw-github", url: "https://example.test" }, + "2026-05-23T00:00:00.000Z", + ), + ); + await upsertRepositoryFromGitHub(env, { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, 123); + await upsertRepositorySettings(env, { + repoFullName: "JSONbored/gittensory", + commentMode: "off", + publicSurface: "off", + autoLabelEnabled: false, + checkRunMode: "off", + gateCheckMode: "enabled", + linkedIssueGateMode: "off", + aiReviewMode: "block", + }); + let gatePatchBody: { conclusion?: string; output?: { title?: string; text?: string } } = {}; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url === "https://api.gittensor.io/miners") { + return Response.json([ + { uid: 7, githubUsername: "confirmed-dev", githubId: "123", totalPrs: 4, totalMergedPrs: 3, totalOpenPrs: 1, totalClosedPrs: 0, totalOpenIssues: 0, totalClosedIssues: 0, totalSolvedIssues: 0, totalValidSolvedIssues: 0, isEligible: true, credibility: 1, eligibleRepoCount: 1 }, + ]); + } + if (url === "https://api.gittensor.io/miners/123") { + return Response.json({ repositories: [{ repositoryFullName: "JSONbored/gittensory", totalPrs: "4", totalMergedPrs: "3", totalOpenPrs: "1", totalClosedPrs: "0", totalOpenIssues: "0", totalClosedIssues: "0", isEligible: true, credibility: "1.000000" }] }); + } + if (url === "https://api.gittensor.io/miners/123/prs") return Response.json([]); + if (url === "https://mirror.gittensor.io/api/v1/miners/123/issues") return Response.json({ issues: [] }); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/commits/aidefect123/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/check-runs/950") && method === "PATCH") { + gatePatchBody = JSON.parse(String(init?.body ?? "{}")) as typeof gatePatchBody; + return Response.json({ id: 950 }); + } + if (url.includes("/check-runs") && method === "POST") return Response.json({ id: 950 }, { status: 201 }); + return new Response("not found", { status: 404 }); + }); + + await processJob(env, { + type: "github-webhook", + deliveryId: "gate-ai-consensus-block", + eventName: "pull_request", + payload: { + action: "opened", + installation: { id: 123, account: { login: "JSONbored", id: 1, type: "User" } }, + repository: { name: "gittensory", full_name: "JSONbored/gittensory", private: false, owner: { login: "JSONbored" } }, + pull_request: { number: 71, title: "Add helper", state: "open", user: { login: "confirmed-dev" }, head: { sha: "aidefect123" }, labels: [], body: "Adds a helper." }, + }, + }); + + expect(gatePatchBody.conclusion).toBe("failure"); + expect(gatePatchBody.output?.title).toContain("AI reviewers agree on a likely critical defect"); + // The AI usage event was recorded for the review (never with key material). + const usage = await env.DB.prepare("select feature, status from ai_usage_events where feature = ?").bind("ai_review_pr").first<{ feature: string; status: string }>(); + expect(usage).toMatchObject({ feature: "ai_review_pr", status: "ok" }); + }); + it("disables the gate from .gittensory.yml (gate.enabled: false) even when repo settings enable it", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem() }); await persistRegistrySnapshot( diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index edf2a99474..02bde68a0e 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -799,12 +799,16 @@ describe("signal coverage edge cases", () => { preflight: buildPreflightResult({ repoFullName: directRepo.fullName, title: "Fix isolated issue", body: "Fixes #99", linkedIssues: [99] }, directRepo, [], [currentPr]), settings: gateSettings, review: { present: true, footerText: "Reviewed by the Acme maintainer bot.", note: "Run npm test before pushing.", fields: { relatedWork: false } }, + aiReview: { notes: "The change is focused.\n\n**Suggestions**\n- Add a test for the edge case." }, }); expect(customizedComment).toContain("Reviewed by the Acme maintainer bot."); // custom footer lead expect(customizedComment).toContain("register to start earning"); // mandatory attribution/earn link kept expect(customizedComment).toContain("Run npm test before pushing."); // intro note expect(customizedComment).not.toContain("| Related work |"); // hidden row expect(customizedComment).toContain("| Gate result |"); // non-hidden rows still rendered + expect(customizedComment).toContain("Gittensory AI review (advisory)"); // AI section rendered + expect(customizedComment).toContain("</details>"); // stray tags escaped, panel structure preserved + expect(customizedComment).toContain("- Add a test for the"); // markdown bullets preserved (not flattened) const advisoryOnlyComment = buildPublicPrIntelligenceComment({ repo: directRepo, From 932f7b8329ae3bd429f05a17a6eae6640b404da8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:56:33 -0700 Subject: [PATCH 3/4] feat(ai-review): BYOK provider keys, encrypted at rest (AES-256-GCM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a maintainer bring their own Anthropic/OpenAI key so the advisory AI review is written by their frontier model instead of free Workers AI. The consensus blocker always stays on the free Workers-AI pair, so BYOK never changes who can be blocked. - crypto.ts: encryptSecret/decryptSecret (AES-256-GCM, PBKDF2-derived key, random 12-byte IV per write) — gittensory's first reversible-encryption primitive. Keyed by a new TOKEN_ENCRYPTION_SECRET worker secret. - New isolated repository_ai_keys table (migration 0027) so the ciphertext is never serialized by the repository-settings surface. Stores ciphertext + iv + key_version + a display-only last4; the plaintext key is never persisted. - repositories.ts: getRepositoryAiKeyStatus (secret-free), upsertRepositoryAiKey (encrypts; throws if the secret is unconfigured), deleteRepositoryAiKey, getDecryptedRepositoryAiKey (decrypts at call time; null on any miss so the review silently falls back to Workers AI). - Write-only internal routes POST/GET/DELETE /v1/internal/repos/:owner/:repo/ai-key — GET returns only { configured, provider, last4, model }, never the key. - Processor decrypts the key only when aiReviewByok is on and passes it to the engine; the key is never logged or placed in usage/audit metadata. The key is encrypted at rest, never returned by any GET, and never reaches a public surface or log line. --- migrations/0027_repository_ai_keys.sql | 12 +++ src/api/routes.ts | 46 ++++++++++ src/db/repositories.ts | 78 ++++++++++++++++- src/db/schema.ts | 16 ++++ src/env.d.ts | 4 + src/queue/processors.ts | 6 +- src/utils/crypto.ts | 42 +++++++++ test/unit/ai-key-byok.test.ts | 116 +++++++++++++++++++++++++ test/unit/ai-review-advisory.test.ts | 31 ++++++- 9 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 migrations/0027_repository_ai_keys.sql create mode 100644 test/unit/ai-key-byok.test.ts diff --git a/migrations/0027_repository_ai_keys.sql b/migrations/0027_repository_ai_keys.sql new file mode 100644 index 0000000000..ef13bdb066 --- /dev/null +++ b/migrations/0027_repository_ai_keys.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS repository_ai_keys ( + repo_full_name TEXT PRIMARY KEY, + provider TEXT NOT NULL, + ciphertext TEXT NOT NULL, + iv TEXT NOT NULL, + key_version INTEGER NOT NULL DEFAULT 1, + model TEXT, + last4 TEXT NOT NULL, + created_by TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/src/api/routes.ts b/src/api/routes.ts index 6a4e6074ff..7be7401054 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -89,6 +89,9 @@ import { upsertContributorEvidence, upsertContributorScoringProfile, upsertRepositorySettings, + getRepositoryAiKeyStatus, + upsertRepositoryAiKey, + deleteRepositoryAiKey, } from "../db/repositories"; import { backfillOpenPullRequestDetails, @@ -527,6 +530,14 @@ const repositorySettingsSchema = z.object({ .default(DEFAULT_COMMAND_AUTHORIZATION_POLICY), }); +// Maintainer BYOK provider key. Write-only: the key is encrypted at rest and never returned. A loose +// shape check (sk-ant-… / sk-…) catches obvious paste errors without coupling to provider key formats. +const repositoryAiKeySchema = z.object({ + provider: z.enum(["anthropic", "openai"]), + key: z.string().trim().min(20).max(400), + model: z.string().trim().min(1).max(120).nullable().optional(), +}); + const contributorIssueDraftGenerateSchema = z.object({ dryRun: z.boolean().optional().default(true), create: z.boolean().optional().default(false), @@ -2462,6 +2473,41 @@ export function createApp() { ); }); + // Maintainer BYOK provider key. GET returns secret-free status only; POST stores it encrypted at rest; + // DELETE removes it. The plaintext key is never logged and never returned. + app.get("/v1/internal/repos/:owner/:repo/ai-key", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + return c.json(await getRepositoryAiKeyStatus(c.env, fullName)); + }); + + app.post("/v1/internal/repos/:owner/:repo/ai-key", async (c) => { + const body = await c.req.json().catch(() => null); + const parsed = repositoryAiKeySchema.safeParse(body); + if (!parsed.success) return c.json({ error: "invalid_ai_key", issues: parsed.error.issues }, 400); + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + try { + const status = await upsertRepositoryAiKey(c.env, { + repoFullName: fullName, + provider: parsed.data.provider, + key: parsed.data.key, + model: parsed.data.model ?? null, + }); + return c.json(status); + } catch (error) { + // The only expected throw is a missing encryption secret — never echo key material in the error. + if (error instanceof Error && error.message === "missing_encryption_secret") { + return c.json({ error: "encryption_unavailable", detail: "TOKEN_ENCRYPTION_SECRET is not configured." }, 503); + } + throw error; + } + }); + + app.delete("/v1/internal/repos/:owner/:repo/ai-key", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + await deleteRepositoryAiKey(c.env, fullName); + return c.json({ configured: false }); + }); + app.get("/v1/internal/repos/:owner/:repo/contribution-policy", async (c) => { const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; const focusManifest = await loadRepoFocusManifest(c.env, fullName, { fetcher: async () => null }); diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 34a0d428e3..d08aa0eb55 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -42,6 +42,7 @@ import { repoSnapshots, repoSyncSegments, repoSyncState, + repositoryAiKeys, repositorySettings, scorePreviews, scoringModelSnapshots, @@ -139,7 +140,7 @@ import type { import type { GittensorContributorSnapshot, OfficialGittensorMinerDetection } from "../gittensor/api"; import { classifyMcpClientVersion, LATEST_RECOMMENDED_MCP_VERSION, MINIMUM_SUPPORTED_MCP_VERSION } from "../services/mcp-compatibility"; import { DEFAULT_COMMAND_AUTHORIZATION_POLICY, normalizeCommandAuthorizationPolicy } from "../settings/command-authorization"; -import { sha256Hex } from "../utils/crypto"; +import { decryptSecret, encryptSecret, sha256Hex } from "../utils/crypto"; import { jsonString, nowIso, parseJson, repoParts } from "../utils/json"; const MAX_STORED_BODY_CHARS = 4000; @@ -514,6 +515,81 @@ export async function upsertRepositorySettings(env: Env, settings: Partial { + const db = getDb(env.DB); + const [row] = await db.select().from(repositoryAiKeys).where(eq(repositoryAiKeys.repoFullName, fullName)).limit(1); + if (!row) return { configured: false }; + return { configured: true, provider: normalizeAiKeyProvider(row.provider), last4: row.last4, model: row.model ?? null }; +} + +/** + * Store (or replace) a repo's BYOK provider key, encrypted at rest. Returns the secret-free status. + * Throws `missing_encryption_secret` when TOKEN_ENCRYPTION_SECRET is not configured — callers must + * surface that rather than store a key in the clear. + */ +export async function upsertRepositoryAiKey( + env: Env, + input: { repoFullName: string; provider: AiKeyProvider; key: string; model?: string | null; createdBy?: string | null }, +): Promise { + const secret = env.TOKEN_ENCRYPTION_SECRET; + if (!secret) throw new Error("missing_encryption_secret"); + const trimmedKey = input.key.trim(); + const { ciphertext, iv, version } = await encryptSecret(trimmedKey, secret); + const last4 = trimmedKey.slice(-4); + const model = input.model?.trim() ? input.model.trim() : null; + const db = getDb(env.DB); + await db + .insert(repositoryAiKeys) + .values({ repoFullName: input.repoFullName, provider: input.provider, ciphertext, iv, keyVersion: version, model, last4, createdBy: input.createdBy ?? null, updatedAt: nowIso() }) + .onConflictDoUpdate({ + target: repositoryAiKeys.repoFullName, + set: { provider: input.provider, ciphertext, iv, keyVersion: version, model, last4, createdBy: input.createdBy ?? null, updatedAt: nowIso() }, + }); + return { configured: true, provider: input.provider, last4, model }; +} + +/** Remove a repo's BYOK key. */ +export async function deleteRepositoryAiKey(env: Env, fullName: string): Promise { + const db = getDb(env.DB); + await db.delete(repositoryAiKeys).where(eq(repositoryAiKeys.repoFullName, fullName)); +} + +/** + * Decrypt a repo's BYOK key for an AI call. Returns null when no key is configured OR the encryption + * secret is unavailable OR decryption fails — so the caller silently falls back to free Workers AI and + * a misconfiguration never blocks the review. The plaintext key must be used immediately and never cached. + */ +export async function getDecryptedRepositoryAiKey(env: Env, fullName: string): Promise { + const secret = env.TOKEN_ENCRYPTION_SECRET; + if (!secret) return null; + const db = getDb(env.DB); + const [row] = await db.select().from(repositoryAiKeys).where(eq(repositoryAiKeys.repoFullName, fullName)).limit(1); + if (!row) return null; + try { + const key = await decryptSecret(row.ciphertext, row.iv, secret); + return { provider: normalizeAiKeyProvider(row.provider), key, model: row.model ?? null }; + } catch { + return null; + } +} + export async function upsertRepoSyncState(env: Env, state: RepoSyncStateRecord): Promise { const db = getDb(env.DB); await db diff --git a/src/db/schema.ts b/src/db/schema.ts index 3ea6b3891e..28e9402871 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -60,6 +60,22 @@ export const repositorySettings = sqliteTable("repository_settings", { updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), }); +// Maintainer BYOK provider keys (Anthropic/OpenAI), encrypted at rest with AES-256-GCM. Isolated in its +// own table so the ciphertext is NEVER serialized by the repository-settings GET surface. The plaintext +// key is never stored; `last4` is a display-only hint derived from the plaintext at write time. +export const repositoryAiKeys = sqliteTable("repository_ai_keys", { + repoFullName: text("repo_full_name").primaryKey(), + provider: text("provider").notNull(), + ciphertext: text("ciphertext").notNull(), + iv: text("iv").notNull(), + keyVersion: integer("key_version").notNull().default(1), + model: text("model"), + last4: text("last4").notNull(), + createdBy: text("created_by"), + createdAt: text("created_at").notNull().default("CURRENT_TIMESTAMP"), + updatedAt: text("updated_at").notNull().default("CURRENT_TIMESTAMP"), +}); + export const repoSyncState = sqliteTable("repo_sync_state", { repoFullName: text("repo_full_name").primaryKey(), status: text("status").notNull().default("never_synced"), diff --git a/src/env.d.ts b/src/env.d.ts index f8c1094c0b..e54c1462a5 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -31,6 +31,10 @@ declare global { GITTENSORY_API_TOKEN: string; GITTENSORY_MCP_TOKEN: string; INTERNAL_JOB_TOKEN: string; + /** AES-256-GCM master secret for maintainer BYOK provider keys (encrypt/decrypt at rest). A Worker + * secret (`wrangler secret put`), never a public var. When absent, BYOK is unavailable and the AI + * review silently falls back to free Workers AI. */ + TOKEN_ENCRYPTION_SECRET?: string; RATE_LIMIT_TRUSTED_PROXIES?: string; RATE_LIMIT_TRUSTED_PROXY_COUNT?: string; } diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 5aaa8865f3..766d4c92f3 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -7,6 +7,7 @@ import { getFreshOfficialMinerDetection, getPullRequest, getRepository, + getDecryptedRepositoryAiKey, getRepositorySettings, listCheckSummaries, listAllIssues, @@ -849,6 +850,9 @@ export async function runAiReviewForAdvisory( ): Promise<{ notes: string } | undefined> { if (args.settings.aiReviewMode === "off" || !args.confirmedContributor || !args.advisory.headSha) return undefined; try { + // BYOK: decrypt the maintainer's provider key only when opted in. Falls back to free Workers AI when + // no key is configured or the encryption secret is unavailable (getDecryptedRepositoryAiKey → null). + const providerKey = args.settings.aiReviewByok ? await getDecryptedRepositoryAiKey(env, args.repoFullName) : null; const files = await listPullRequestFiles(env, args.repoFullName, args.pr.number); const result = await runGittensoryAiReview(env, { repoFullName: args.repoFullName, @@ -858,7 +862,7 @@ export async function runAiReviewForAdvisory( diff: buildAiReviewDiff(files), actor: args.author, mode: args.settings.aiReviewMode === "block" ? "block" : "advisory", - providerKey: null, + providerKey, }); if (result.status !== "ok") return undefined; if (result.consensusDefect) { diff --git a/src/utils/crypto.ts b/src/utils/crypto.ts index 559ee81758..b5bc418b52 100644 --- a/src/utils/crypto.ts +++ b/src/utils/crypto.ts @@ -41,6 +41,48 @@ function hexToBytes(hex: string): Uint8Array { return bytes; } +// ─── Reversible secret encryption (AES-256-GCM) ───────────────────────────────────────────────── +// Used for maintainer BYOK provider keys (Anthropic/OpenAI) that MUST be recoverable in plaintext at +// AI-call time. The AES key is derived from the worker secret TOKEN_ENCRYPTION_SECRET via PBKDF2; a +// fresh random 12-byte IV is used per encryption so ciphertexts are unique and the GCM tag authenticates +// them. The plaintext key is never persisted, never logged, and never returned from the API. +const SECRET_KDF_SALT = new TextEncoder().encode("gittensory-secret-encryption-v1"); +const SECRET_KEY_VERSION = 1; + +async function deriveSecretAesKey(keyMaterial: string): Promise { + const baseKey = await crypto.subtle.importKey("raw", new TextEncoder().encode(keyMaterial), "PBKDF2", false, ["deriveKey"]); + return crypto.subtle.deriveKey( + { name: "PBKDF2", salt: SECRET_KDF_SALT, iterations: 100_000, hash: "SHA-256" }, + baseKey, + { name: "AES-GCM", length: 256 }, + false, + ["encrypt", "decrypt"], + ); +} + +/** Encrypt a secret with AES-256-GCM. Returns base64 ciphertext (incl. auth tag) + base64 IV + version. */ +export async function encryptSecret(plaintext: string, keyMaterial: string): Promise<{ ciphertext: string; iv: string; version: number }> { + if (!keyMaterial) throw new Error("missing_encryption_secret"); + const key = await deriveSecretAesKey(keyMaterial); + const iv = crypto.getRandomValues(new Uint8Array(12)); + const encrypted = await crypto.subtle.encrypt({ name: "AES-GCM", iv }, key, new TextEncoder().encode(plaintext)); + return { ciphertext: base64Encode(new Uint8Array(encrypted)), iv: base64Encode(iv), version: SECRET_KEY_VERSION }; +} + +/** Decrypt a secret produced by {@link encryptSecret}. Throws if the secret/IV/ciphertext do not match. */ +export async function decryptSecret(ciphertext: string, iv: string, keyMaterial: string): Promise { + if (!keyMaterial) throw new Error("missing_encryption_secret"); + const key = await deriveSecretAesKey(keyMaterial); + const decrypted = await crypto.subtle.decrypt({ name: "AES-GCM", iv: base64ToBytes(iv) }, key, base64ToBytes(ciphertext)); + return new TextDecoder().decode(decrypted); +} + +function base64Encode(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary); +} + export function base64UrlEncode(input: Uint8Array | string): string { const bytes = typeof input === "string" ? new TextEncoder().encode(input) : input; let binary = ""; diff --git a/test/unit/ai-key-byok.test.ts b/test/unit/ai-key-byok.test.ts new file mode 100644 index 0000000000..f79d06c8ba --- /dev/null +++ b/test/unit/ai-key-byok.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { createApp } from "../../src/api/routes"; +import { decryptSecret, encryptSecret } from "../../src/utils/crypto"; +import { deleteRepositoryAiKey, getDecryptedRepositoryAiKey, getRepositoryAiKeyStatus, upsertRepositoryAiKey } from "../../src/db/repositories"; +import { createTestEnv } from "../helpers/d1"; + +const SECRET = "unit-test-encryption-secret-at-least-32-bytes-long"; + +describe("encryptSecret / decryptSecret (AES-256-GCM)", () => { + it("round-trips a secret and produces a fresh IV each time", async () => { + const a = await encryptSecret("sk-ant-supersecret", SECRET); + const b = await encryptSecret("sk-ant-supersecret", SECRET); + expect(a.iv).not.toBe(b.iv); // random IV per encryption + expect(a.ciphertext).not.toBe(b.ciphertext); + expect(a.version).toBe(1); + await expect(decryptSecret(a.ciphertext, a.iv, SECRET)).resolves.toBe("sk-ant-supersecret"); + }); + + it("fails to decrypt with the wrong secret and throws without a key", async () => { + const { ciphertext, iv } = await encryptSecret("sk-secret", SECRET); + await expect(decryptSecret(ciphertext, iv, "a-different-secret-of-sufficient-length-here")).rejects.toThrow(); + await expect(encryptSecret("x", "")).rejects.toThrow("missing_encryption_secret"); + await expect(decryptSecret(ciphertext, iv, "")).rejects.toThrow("missing_encryption_secret"); + }); +}); + +describe("repository BYOK key storage", () => { + it("stores an encrypted key, exposes only secret-free status, and decrypts at call time", async () => { + const env = createTestEnv({ TOKEN_ENCRYPTION_SECRET: SECRET }); + await expect(getRepositoryAiKeyStatus(env, "acme/widgets")).resolves.toEqual({ configured: false }); + + const status = await upsertRepositoryAiKey(env, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-abc123XYZ7890", model: "claude-3-5-sonnet-latest", createdBy: "maintainer" }); + expect(status).toEqual({ configured: true, provider: "anthropic", last4: "7890", model: "claude-3-5-sonnet-latest" }); + + // Status surface never includes the key or ciphertext. + const fetched = await getRepositoryAiKeyStatus(env, "acme/widgets"); + expect(JSON.stringify(fetched)).not.toContain("sk-ant"); + expect(fetched).toMatchObject({ configured: true, last4: "7890" }); + + // Decrypt only happens at call time. + await expect(getDecryptedRepositoryAiKey(env, "acme/widgets")).resolves.toEqual({ provider: "anthropic", key: "sk-ant-abc123XYZ7890", model: "claude-3-5-sonnet-latest" }); + + // The persisted row stores ciphertext, never the plaintext key. + const row = await env.DB.prepare("select ciphertext, iv, last4 from repository_ai_keys where repo_full_name = ?").bind("acme/widgets").first<{ ciphertext: string; iv: string; last4: string }>(); + expect(row?.ciphertext).not.toContain("sk-ant"); + expect(row?.last4).toBe("7890"); + }); + + it("replaces a key on re-set and removes it on delete", async () => { + const env = createTestEnv({ TOKEN_ENCRYPTION_SECRET: SECRET }); + await upsertRepositoryAiKey(env, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-first0000", model: null }); + await upsertRepositoryAiKey(env, { repoFullName: "acme/widgets", provider: "openai", key: "sk-openai-second1111", model: null }); + await expect(getRepositoryAiKeyStatus(env, "acme/widgets")).resolves.toMatchObject({ configured: true, provider: "openai", last4: "1111" }); + await deleteRepositoryAiKey(env, "acme/widgets"); + await expect(getRepositoryAiKeyStatus(env, "acme/widgets")).resolves.toEqual({ configured: false }); + await expect(getDecryptedRepositoryAiKey(env, "acme/widgets")).resolves.toBeNull(); + }); + + it("refuses to store a key and cannot decrypt without the encryption secret", async () => { + const noSecret = createTestEnv({}); + await expect(upsertRepositoryAiKey(noSecret, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-xyz" })).rejects.toThrow("missing_encryption_secret"); + // A row encrypted under SECRET cannot be decrypted when the env has no secret → null (falls back). + const withSecret = createTestEnv({ TOKEN_ENCRYPTION_SECRET: SECRET }); + await upsertRepositoryAiKey(withSecret, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-abc1234567" }); + const sameDbNoSecret = { ...withSecret, TOKEN_ENCRYPTION_SECRET: undefined } as unknown as Env; + await expect(getDecryptedRepositoryAiKey(sameDbNoSecret, "acme/widgets")).resolves.toBeNull(); + // A row that cannot be decrypted (wrong secret) → null, not a throw. + const wrongSecret = { ...withSecret, TOKEN_ENCRYPTION_SECRET: "totally-different-secret-32-bytes-min" } as unknown as Env; + await expect(getDecryptedRepositoryAiKey(wrongSecret, "acme/widgets")).resolves.toBeNull(); + }); +}); + +describe("BYOK API routes", () => { + function authHeaders(env: Env) { + return { authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}`, "content-type": "application/json" }; + } + + it("POST stores, GET returns secret-free status, DELETE removes — key never echoed", async () => { + const app = createApp(); + const env = createTestEnv({ TOKEN_ENCRYPTION_SECRET: SECRET }); + + const post = await app.request( + "/v1/internal/repos/acme/widgets/ai-key", + { method: "POST", headers: authHeaders(env), body: JSON.stringify({ provider: "anthropic", key: "sk-ant-route-key-7777", model: "claude-3-5-sonnet-latest" }) }, + env, + ); + expect(post.status).toBe(200); + const postBody = await post.json(); + expect(postBody).toMatchObject({ configured: true, provider: "anthropic", last4: "7777" }); + expect(JSON.stringify(postBody)).not.toContain("sk-ant"); + + const get = await app.request("/v1/internal/repos/acme/widgets/ai-key", { headers: authHeaders(env) }, env); + expect(await get.json()).toMatchObject({ configured: true, last4: "7777" }); + + const del = await app.request("/v1/internal/repos/acme/widgets/ai-key", { method: "DELETE", headers: authHeaders(env) }, env); + expect(await del.json()).toEqual({ configured: false }); + const getAfter = await app.request("/v1/internal/repos/acme/widgets/ai-key", { headers: authHeaders(env) }, env); + expect(await getAfter.json()).toEqual({ configured: false }); + }); + + it("rejects an invalid key payload and reports when encryption is unavailable", async () => { + const app = createApp(); + const env = createTestEnv({ TOKEN_ENCRYPTION_SECRET: SECRET }); + const bad = await app.request("/v1/internal/repos/acme/widgets/ai-key", { method: "POST", headers: authHeaders(env), body: JSON.stringify({ provider: "anthropic", key: "short" }) }, env); + expect(bad.status).toBe(400); + + const noSecretEnv = createTestEnv({}); + const unavailable = await app.request( + "/v1/internal/repos/acme/widgets/ai-key", + { method: "POST", headers: authHeaders(noSecretEnv), body: JSON.stringify({ provider: "openai", key: "sk-openai-valid-key-123456" }) }, + noSecretEnv, + ); + expect(unavailable.status).toBe(503); + expect(await unavailable.json()).toMatchObject({ error: "encryption_unavailable" }); + }); +}); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index d9db737fb7..79a1708fe6 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -1,8 +1,13 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { buildAiReviewDiff, runAiReviewForAdvisory } from "../../src/queue/processors"; +import { upsertRepositoryAiKey } from "../../src/db/repositories"; import type { Advisory, PullRequestFileRecord, RepositorySettings } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; +afterEach(() => { + vi.unstubAllGlobals(); +}); + function fileRecord(over: Partial & { path: string }): PullRequestFileRecord { return { repoFullName: "acme/widgets", pullNumber: 3, status: "modified", additions: 1, deletions: 0, changes: 1, payload: {}, ...over }; } @@ -137,6 +142,30 @@ describe("runAiReviewForAdvisory", () => { expect(result).toBeUndefined(); }); + it("uses the maintainer's BYOK provider key when aiReviewByok is on and a key is configured", async () => { + const env = createTestEnv({ + AI: { run: async () => ({ response: notesOnlyJson() }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + TOKEN_ENCRYPTION_SECRET: "advisory-test-encryption-secret-32bytes", + }); + await upsertRepositoryAiKey(env, { repoFullName: "acme/widgets", provider: "anthropic", key: "sk-ant-byok-key-9999", model: null }); + const fetchMock = vi.fn(async (_url: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({ content: [{ type: "text", text: notesOnlyJson() }] }), { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const result = await runAiReviewForAdvisory(env, { + settings: { aiReviewMode: "advisory", aiReviewByok: true } as RepositorySettings, + advisory: advisory(), + repoFullName: "acme/widgets", + pr, + author: "alice", + confirmedContributor: true, + }); + expect(result?.notes).toContain("Add a test."); + // Advisory write-up went to the BYOK provider, not Workers AI. + expect(fetchMock.mock.calls[0]?.[0]).toBe("https://api.anthropic.com/v1/messages"); + }); + it("is fail-safe: a thrown error (e.g. broken DB) yields no finding and no notes", async () => { const adv = advisory(); const env = aiEnv(async () => ({ response: defectJson() })); From 1d335af2cb09e18ca6015510e863a2944180059a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 13 Jun 2026 14:58:13 -0700 Subject: [PATCH 4/4] docs(ai-review): document aiReview config and reconcile BYOK with the PAT policy - .gittensory.yml + bundled manifest: a commented gate.aiReview example (mode + byok), kept byte-identical per the alignment test. - CONTRIBUTING.md: clarify that a maintainer's own opt-in AI-provider key (encrypted, write-only, never returned) is a distinct credential class from the banned contributor GitHub PATs. --- .gittensory.yml | 3 +++ CONTRIBUTING.md | 5 ++++- src/config/gittensory-repo-focus-manifest.ts | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gittensory.yml b/.gittensory.yml index 87a4992086..60b14dbe42 100644 --- a/.gittensory.yml +++ b/.gittensory.yml @@ -42,6 +42,9 @@ gate: readiness: mode: advisory # block | advisory | off — readiness-score floor minScore: 60 + # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) + # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect + # byok: false # use a maintainer Anthropic/OpenAI key for the write-up; consensus stays free Workers AI publicNotes: - Prefer backend Workers, MCP, GitHub App, registry, and scoring work when scope allows. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce58d93371..1cecf5b414 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,10 @@ Do not open PRs for: - Auto-closing, auto-merging, rewriting contributor work, or applying labels outside the explicit confirmed-miner GitHub App policy. - Storing contributor GitHub PATs or adding non-GitHub identity providers. Browser auth is GitHub - OAuth; CLI/MCP auth is GitHub Device Flow. + OAuth; CLI/MCP auth is GitHub Device Flow. (A maintainer's own optional AI-provider key for BYOK AI + review is a different credential class — it is the repo owner's LLM-inference key, not a GitHub + identity credential — and is allowed: it is opt-in, encrypted at rest, write-only, and never returned + or logged.) - Large dependency major upgrades bundled with unrelated product changes. - Changelog edits in ordinary feature/fix PRs. Changelogs are updated during release prep. - Low-effort reward-farming changes, spam, generated bulk edits, or PRs that do not explain the diff --git a/src/config/gittensory-repo-focus-manifest.ts b/src/config/gittensory-repo-focus-manifest.ts index f6beb1699e..7b5abf7c5a 100644 --- a/src/config/gittensory-repo-focus-manifest.ts +++ b/src/config/gittensory-repo-focus-manifest.ts @@ -46,6 +46,9 @@ gate: readiness: mode: advisory # block | advisory | off — readiness-score floor minScore: 60 + # aiReview: # opt-in AI maintainer review (off by default; needs the AI flags enabled) + # mode: advisory # block | advisory | off — block only blocks on a dual-model consensus defect + # byok: false # use a maintainer Anthropic/OpenAI key for the write-up; consensus stays free Workers AI publicNotes: - Prefer backend Workers, MCP, GitHub App, registry, and scoring work when scope allows.