From 46d33e50f677f89a7547409fc8b1da69e9128dcc Mon Sep 17 00:00:00 2001 From: bittoby <218712309+bittoby@users.noreply.github.com> Date: Mon, 15 Jun 2026 04:48:17 +0000 Subject: [PATCH 1/4] feat(api): add public-safe README status badge with badgeEnabled opt-in (#541) --- migrations/0037_badge_enabled.sql | 3 + src/api/badge.ts | 110 ++++++++++++++++++++++++ src/api/routes.ts | 41 +++++++++ src/db/repositories.ts | 5 ++ src/db/schema.ts | 1 + src/services/public-repo-quality.ts | 78 +++++++++++++++++ src/types.ts | 3 + test/integration/api.test.ts | 56 ++++++++++++ test/unit/badge.test.ts | 87 +++++++++++++++++++ test/unit/public-repo-quality.test.ts | 117 ++++++++++++++++++++++++++ 10 files changed, 501 insertions(+) create mode 100644 migrations/0037_badge_enabled.sql create mode 100644 src/api/badge.ts create mode 100644 src/services/public-repo-quality.ts create mode 100644 test/unit/badge.test.ts create mode 100644 test/unit/public-repo-quality.test.ts diff --git a/migrations/0037_badge_enabled.sql b/migrations/0037_badge_enabled.sql new file mode 100644 index 0000000000..ba3227efb0 --- /dev/null +++ b/migrations/0037_badge_enabled.sql @@ -0,0 +1,3 @@ +-- #541: opt-in flag for the public README status badge. Default 0 (off) — the unauthenticated badge +-- endpoint only serves whitelisted metrics for installed repos that have explicitly opted in. +ALTER TABLE repository_settings ADD COLUMN badge_enabled INTEGER NOT NULL DEFAULT 0; diff --git a/src/api/badge.ts b/src/api/badge.ts new file mode 100644 index 0000000000..77c1273592 --- /dev/null +++ b/src/api/badge.ts @@ -0,0 +1,110 @@ +import type { PublicRepoQuality, QueueHealthLevel } from "../services/public-repo-quality"; + +// Self-rendered README status badge (#541). Renders ONLY the public-safe whitelisted metrics from +// `PublicRepoQuality` — no external badge service, no contributor/reward/trust data. All text is XML-escaped +// before it reaches the SVG so the unauthenticated, embeddable surface cannot be turned into an injection +// vector even if upstream values ever change shape. + +const LABEL = "gittensory"; + +const QUEUE_COLORS: Record = { + low: "#3fb950", + medium: "#d29922", + high: "#db6d28", + critical: "#f85149", +}; + +const LOW_REAL_CONTRIBUTION_PCT = 50; +const UNAVAILABLE_COLOR = "#9e9e9e"; + +export type ShieldsBadge = { + schemaVersion: 1; + label: string; + message: string; + color: string; + cacheSeconds: number; +}; + +export function buildBadgeMessage(quality: PublicRepoQuality): string { + const real = quality.realContributionPct === null ? "real n/a" : `${quality.realContributionPct}% real`; + const merge = + quality.medianTimeToMergeHours === null ? "merge n/a" : `merge ${formatDuration(quality.medianTimeToMergeHours)}`; + return `${real} · ${merge} · queue ${quality.queueHealthLevel}`; +} + +export function buildBadgeColor(quality: PublicRepoQuality): string { + // Color tracks queue health, but a low real-contribution share dominates the signal. + if (quality.realContributionPct !== null && quality.realContributionPct < LOW_REAL_CONTRIBUTION_PCT) { + return QUEUE_COLORS.high; + } + return QUEUE_COLORS[quality.queueHealthLevel]; +} + +export function buildShieldsBadge(quality: PublicRepoQuality, cacheSeconds: number): ShieldsBadge { + return { + schemaVersion: 1, + label: LABEL, + message: buildBadgeMessage(quality), + color: buildBadgeColor(quality), + cacheSeconds, + }; +} + +export function renderBadgeSvg(quality: PublicRepoQuality): string { + return renderFlatBadge(LABEL, buildBadgeMessage(quality), buildBadgeColor(quality)); +} + +export function renderUnavailableBadgeSvg(): string { + return renderFlatBadge(LABEL, "unavailable", UNAVAILABLE_COLOR); +} + +function formatDuration(hours: number): string { + if (hours < 1) return "<1h"; + if (hours < 48) return `${Math.round(hours)}h`; + return `${Math.round(hours / 24)}d`; +} + +// Minimal flat ("shields"-style) badge. Widths are approximated from character count; exactness is not +// required for a README badge and keeps the renderer dependency-free. +function renderFlatBadge(label: string, message: string, color: string): string { + const labelText = escapeXml(label); + const messageText = escapeXml(message); + const labelWidth = textWidth(label); + const messageWidth = textWidth(message); + const totalWidth = labelWidth + messageWidth; + const labelMid = labelWidth / 2; + const messageMid = labelWidth + messageWidth / 2; + return [ + ``, + `${labelText}: ${messageText}`, + ``, + ``, + ``, + ``, + `${labelText}`, + `${messageText}`, + ``, + ].join(""); +} + +function textWidth(text: string): number { + // ~6.5px per character + 10px horizontal padding, clamped to a sane minimum. + return Math.max(40, Math.round(text.length * 6.5) + 10); +} + +export function escapeXml(value: string): string { + return value.replace(/[&<>"']/g, (char) => { + switch (char) { + case "&": + return "&"; + case "<": + return "<"; + case ">": + return ">"; + case '"': + return """; + default: + return "'"; + } + }); +} diff --git a/src/api/routes.ts b/src/api/routes.ts index d89041e58b..ee99c275bc 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -156,6 +156,8 @@ import { import { buildOperatorDashboardPayload } from "../services/operator-dashboard"; import { buildSelfDogfoodRegistrationPack, resolveSelfDogfoodRepoFullName } from "../services/self-dogfood-registration-pack"; import { buildSubnetInterfaceDescriptor } from "../services/subnet-interface"; +import { buildPublicRepoQuality, type PublicRepoQuality } from "../services/public-repo-quality"; +import { buildShieldsBadge, renderBadgeSvg, renderUnavailableBadgeSvg } from "./badge"; import { buildWeeklyValueReport, formatWeeklyValueReportMarkdown, @@ -236,6 +238,18 @@ import { errorMessage, nowIso } from "../utils/json"; type AppBindings = { Bindings: Env }; type AppContext = Context; +// Resolves the public README badge metrics for a repo, enforcing the two gates in one place: the repo must +// be installed AND have opted in via `badgeEnabled`. Returns null (→ a benign "unavailable" badge) for any +// repo that is unknown, uninstalled, or has not opted in — so no metrics are ever served otherwise. +async function loadPublicRepoBadge(env: Env, owner: string, repo: string): Promise { + const repository = await getRepository(env, `${owner}/${repo}`); + if (!repository || !repository.isInstalled) return null; + const settings = await getRepositorySettings(env, repository.fullName); + if (!settings.badgeEnabled) return null; + const pullRequests = await listPullRequests(env, repository.fullName); + return buildPublicRepoQuality(pullRequests); +} + async function recordRouteProductUsage( c: AppContext, event: { @@ -557,6 +571,7 @@ const repositorySettingsSchema = z.object({ requireLinkedIssue: z.boolean().default(false), backfillEnabled: z.boolean().default(true), privateTrustEnabled: z.boolean().default(true), + badgeEnabled: z.boolean().default(false), commandAuthorization: z .object({ default: z.array(z.enum(["maintainer", "collaborator", "pr_author", "confirmed_miner"])).max(4).optional(), @@ -722,6 +737,30 @@ export function createApp() { } }); + // Public-safe README status badge (#541). Unauthenticated and embeddable: it serves ONLY whitelisted, + // repo-level metrics, and ONLY for installed repos that opted in via the `badgeEnabled` setting. Excluded + // from requiresApiToken above; aggressively cached + stale-while-revalidate like the public stats route. + app.get("/v1/public/repos/:owner/:repo/badge.svg", async (c) => { + const quality = await loadPublicRepoBadge(c.env, c.req.param("owner"), c.req.param("repo")); + c.header("Content-Type", "image/svg+xml; charset=utf-8"); + if (!quality) { + c.header("Cache-Control", "public, max-age=300"); + return c.body(renderUnavailableBadgeSvg(), 404); + } + c.header("Cache-Control", "public, max-age=600, stale-while-revalidate=86400"); + return c.body(renderBadgeSvg(quality)); + }); + + app.get("/v1/public/repos/:owner/:repo/badge.json", async (c) => { + const quality = await loadPublicRepoBadge(c.env, c.req.param("owner"), c.req.param("repo")); + if (!quality) { + c.header("Cache-Control", "public, max-age=300"); + return c.json({ schemaVersion: 1, label: "gittensory", message: "unavailable", color: "#9e9e9e", cacheSeconds: 300 }, 404); + } + c.header("Cache-Control", "public, max-age=600, stale-while-revalidate=86400"); + return c.json(buildShieldsBadge(quality, 600)); + }); + app.get("/v1/auth/github/start", async (c) => { try { const start = await startGitHubWebOAuth(c.env, c.req.url, c.req.query("returnTo")); @@ -2710,6 +2749,7 @@ export function createApp() { requireLinkedIssue: parsed.data.requireLinkedIssue, backfillEnabled: parsed.data.backfillEnabled, privateTrustEnabled: parsed.data.privateTrustEnabled, + badgeEnabled: parsed.data.badgeEnabled, commandAuthorization: normalizeCommandAuthorizationPolicy(parsed.data.commandAuthorization).policy, }), ); @@ -4255,6 +4295,7 @@ function requiresApiToken(path: string): boolean { if (path === "/health") return false; if (path === "/v1/mcp/compatibility") return false; if (/^\/v1\/public\/github\/repos\/[^/]+\/[^/]+\/stats$/.test(path)) return false; + if (/^\/v1\/public\/repos\/[^/]+\/[^/]+\/badge\.(svg|json)$/.test(path)) return false; if (path === "/v1/public/subnet-interface") return false; if (path === "/openapi.json") return false; if (path === "/mcp") return false; diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5d94002c85..9d93ffbcfb 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -415,6 +415,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise requireLinkedIssue: false, backfillEnabled: true, privateTrustEnabled: true, + badgeEnabled: false, commandAuthorization: normalizeCommandAuthorizationPolicy(DEFAULT_COMMAND_AUTHORIZATION_POLICY).policy, }; } @@ -446,6 +447,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise requireLinkedIssue: row.requireLinkedIssue, backfillEnabled: row.backfillEnabled, privateTrustEnabled: row.privateTrustEnabled, + badgeEnabled: row.badgeEnabled, commandAuthorization: parseCommandAuthorizationPolicy(row.commandAuthorizationJson), createdAt: row.createdAt, updatedAt: row.updatedAt, @@ -481,6 +483,7 @@ export async function upsertRepositorySettings(env: Env, settings: Partial nowIso()), updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()), diff --git a/src/services/public-repo-quality.ts b/src/services/public-repo-quality.ts new file mode 100644 index 0000000000..83ebb207a0 --- /dev/null +++ b/src/services/public-repo-quality.ts @@ -0,0 +1,78 @@ +import type { PullRequestRecord } from "../types"; + +// Public-safe repository quality metrics for the unauthenticated README badge (#541). +// +// HARD whitelist: this module derives ONLY three coarse, repo-level, public-safe metrics from cached +// pull-request records — median time-to-merge, the share of non-slop merged contributions, and a coarse +// queue-health level. It never reads or exposes contributor-level data, reward/trust values, or private +// scoreability context. Pure and deterministic (clock injected) so the badge surface stays auditable. + +export type QueueHealthLevel = "low" | "medium" | "high" | "critical"; + +export type PublicRepoQuality = { + /** Median hours from PR open to merge across known merged PRs. `null` when none are known. */ + medianTimeToMergeHours: number | null; + /** Share (0-100) of *assessed* merged PRs whose slop band is clean/low. `null` when none assessed. */ + realContributionPct: number | null; + queueHealthLevel: QueueHealthLevel; + /** Counts only — included for transparency, never contributor-level detail. */ + mergedSampleSize: number; + assessedSampleSize: number; +}; + +const STALE_OPEN_PR_DAYS = 14; +const MS_PER_HOUR = 3_600_000; +const MS_PER_DAY = 86_400_000; +const NON_SLOP_BANDS: ReadonlySet = new Set(["clean", "low"]); + +export function buildPublicRepoQuality(pullRequests: PullRequestRecord[], now: number = Date.now()): PublicRepoQuality { + const merged = pullRequests.filter(isMergedPullRequest); + const mergeDurations = merged + .map(mergeDurationHours) + .filter((hours): hours is number => hours !== null); + const assessed = merged.filter((pr) => typeof pr.slopBand === "string" && pr.slopBand.trim().length > 0); + const nonSlop = assessed.filter((pr) => NON_SLOP_BANDS.has((pr.slopBand as string).toLowerCase())); + + return { + medianTimeToMergeHours: mergeDurations.length > 0 ? Math.round(median(mergeDurations)) : null, + realContributionPct: assessed.length > 0 ? Math.round((nonSlop.length / assessed.length) * 100) : null, + queueHealthLevel: resolveQueueHealthLevel(pullRequests, now), + mergedSampleSize: merged.length, + assessedSampleSize: assessed.length, + }; +} + +function isMergedPullRequest(pr: PullRequestRecord): boolean { + return Boolean(pr.mergedAt) || pr.state.toLowerCase() === "merged"; +} + +function mergeDurationHours(pr: PullRequestRecord): number | null { + if (!pr.mergedAt || !pr.createdAt) return null; + const merged = Date.parse(pr.mergedAt); + const created = Date.parse(pr.createdAt); + if (!Number.isFinite(merged) || !Number.isFinite(created) || merged < created) return null; + return (merged - created) / MS_PER_HOUR; +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) return ((sorted[mid - 1] as number) + (sorted[mid] as number)) / 2; + return sorted[mid] as number; +} + +// Coarse, public-safe queue level derived only from open-PR volume and staleness — deliberately simpler +// than the internal QueueHealth signal so no private-derived value reaches this unauthenticated surface. +function resolveQueueHealthLevel(pullRequests: PullRequestRecord[], now: number): QueueHealthLevel { + const open = pullRequests.filter((pr) => pr.state.toLowerCase() === "open"); + const openCount = open.length; + const staleCount = open.filter((pr) => { + const stamp = Date.parse(pr.updatedAt ?? pr.createdAt ?? ""); + return Number.isFinite(stamp) && (now - stamp) / MS_PER_DAY >= STALE_OPEN_PR_DAYS; + }).length; + + if (openCount >= 50 || staleCount >= 20) return "critical"; + if (openCount >= 20 || staleCount >= 8) return "high"; + if (openCount >= 5 || staleCount >= 2) return "medium"; + return "low"; +} diff --git a/src/types.ts b/src/types.ts index f16a5d1ca6..c10efbd031 100644 --- a/src/types.ts +++ b/src/types.ts @@ -444,6 +444,9 @@ export type RepositorySettings = { requireLinkedIssue: boolean; backfillEnabled: boolean; privateTrustEnabled: boolean; + /** Opt-in for the public, unauthenticated README status badge (#541). Always populated by the DB layer + * (default false); optional so existing settings fixtures/callers need not be touched. */ + badgeEnabled?: boolean | undefined; commandAuthorization?: RepositoryCommandAuthorizationPolicy | undefined; createdAt?: string | null | undefined; updatedAt?: string | null | undefined; diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index faad53c53e..e62362016c 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -193,6 +193,62 @@ describe("api routes", () => { await expect(unavailable.json()).resolves.toMatchObject({ error: "github_repo_stats_unavailable" }); }); + it("serves the public README badge only for installed, opted-in repos (#541)", async () => { + const app = createApp(); + const env = createTestEnv(); + + // Installed + opted in, with assessed merged PRs. + await upsertRepositoryFromGitHub(env, { name: "badged", full_name: "acme/badged", private: false, owner: { login: "acme" }, default_branch: "main" }, 555); + await upsertRepositorySettings(env, { repoFullName: "acme/badged", badgeEnabled: true }); + await upsertPullRequestFromGitHub(env, "acme/badged", { number: 1, title: "Feature", state: "merged", created_at: "2026-06-01T00:00:00Z", merged_at: "2026-06-01T04:00:00Z", labels: [] }); + await upsertPullRequestFromGitHub(env, "acme/badged", { number: 2, title: "Slop", state: "merged", created_at: "2026-06-02T00:00:00Z", merged_at: "2026-06-02T06:00:00Z", labels: [] }); + await updatePullRequestSlopAssessment(env, "acme/badged", 1, { slopRisk: 0, slopBand: "clean" }); + await updatePullRequestSlopAssessment(env, "acme/badged", 2, { slopRisk: 80, slopBand: "high" }); + + const svg = await app.request("/v1/public/repos/acme/badged/badge.svg", {}, env); + expect(svg.status).toBe(200); + expect(svg.headers.get("content-type")).toContain("image/svg+xml"); + expect(svg.headers.get("cache-control")).toContain("stale-while-revalidate"); + const svgBody = await svg.text(); + expect(svgBody.startsWith(" { + const app = createApp(); + const env = createTestEnv(); + const response = await app.request( + "/v1/internal/repos/acme/badged/settings", + { method: "POST", headers: { authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}`, "content-type": "application/json" }, body: JSON.stringify({ badgeEnabled: true }) }, + env, + ); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ repoFullName: "acme/badged", badgeEnabled: true }); + }); + it("rejects invalid public GitHub repo stats paths before calling GitHub", async () => { const app = createApp(); const env = createTestEnv(); diff --git a/test/unit/badge.test.ts b/test/unit/badge.test.ts new file mode 100644 index 0000000000..2f3eb9d253 --- /dev/null +++ b/test/unit/badge.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + buildBadgeColor, + buildBadgeMessage, + buildShieldsBadge, + escapeXml, + renderBadgeSvg, + renderUnavailableBadgeSvg, +} from "../../src/api/badge"; +import type { PublicRepoQuality } from "../../src/services/public-repo-quality"; + +function quality(overrides: Partial = {}): PublicRepoQuality { + return { + medianTimeToMergeHours: 30, + realContributionPct: 92, + queueHealthLevel: "low", + mergedSampleSize: 10, + assessedSampleSize: 8, + ...overrides, + }; +} + +describe("buildBadgeMessage", () => { + it("summarizes the whitelisted metrics", () => { + expect(buildBadgeMessage(quality())).toBe("92% real · merge 30h · queue low"); + }); + + it("renders n/a for missing metrics and formats duration by magnitude", () => { + expect(buildBadgeMessage(quality({ realContributionPct: null, medianTimeToMergeHours: null }))).toBe( + "real n/a · merge n/a · queue low", + ); + expect(buildBadgeMessage(quality({ medianTimeToMergeHours: 0 }))).toContain("merge <1h"); + expect(buildBadgeMessage(quality({ medianTimeToMergeHours: 72 }))).toContain("merge 3d"); + }); +}); + +describe("buildBadgeColor", () => { + it("tracks queue health when contribution quality is healthy", () => { + expect(buildBadgeColor(quality({ queueHealthLevel: "low" }))).toBe("#3fb950"); + expect(buildBadgeColor(quality({ queueHealthLevel: "medium" }))).toBe("#d29922"); + expect(buildBadgeColor(quality({ queueHealthLevel: "critical" }))).toBe("#f85149"); + }); + + it("downgrades the color when the real-contribution share is low", () => { + expect(buildBadgeColor(quality({ queueHealthLevel: "low", realContributionPct: 40 }))).toBe("#db6d28"); + }); + + it("uses queue color when the contribution share is unknown", () => { + expect(buildBadgeColor(quality({ queueHealthLevel: "low", realContributionPct: null }))).toBe("#3fb950"); + }); +}); + +describe("buildShieldsBadge", () => { + it("emits a shields endpoint payload", () => { + expect(buildShieldsBadge(quality(), 600)).toEqual({ + schemaVersion: 1, + label: "gittensory", + message: "92% real · merge 30h · queue low", + color: "#3fb950", + cacheSeconds: 600, + }); + }); +}); + +describe("renderBadgeSvg", () => { + it("renders a valid SVG carrying the label and message", () => { + const svg = renderBadgeSvg(quality()); + expect(svg.startsWith(" { + const svg = renderUnavailableBadgeSvg(); + expect(svg).toContain("unavailable"); + expect(svg.startsWith(" { + it("escapes all XML-significant characters", () => { + expect(escapeXml("&<>\"'")).toBe("&<>"'"); + expect(escapeXml("safe text 92%")).toBe("safe text 92%"); + }); +}); diff --git a/test/unit/public-repo-quality.test.ts b/test/unit/public-repo-quality.test.ts new file mode 100644 index 0000000000..1cd76bf7a9 --- /dev/null +++ b/test/unit/public-repo-quality.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { buildPublicRepoQuality } from "../../src/services/public-repo-quality"; +import type { PullRequestRecord } from "../../src/types"; + +const NOW = Date.parse("2026-06-15T00:00:00.000Z"); + +function pr(overrides: Partial = {}): PullRequestRecord { + return { + repoFullName: "acme/widgets", + number: 1, + title: "PR", + state: "merged", + labels: [], + linkedIssues: [], + ...overrides, + }; +} + +function merged(createdAt: string, mergedAt: string, slopBand?: string): PullRequestRecord { + return pr({ state: "merged", createdAt, mergedAt, ...(slopBand ? { slopBand } : {}) }); +} + +describe("buildPublicRepoQuality", () => { + it("returns public-safe defaults for an empty repo", () => { + expect(buildPublicRepoQuality([], NOW)).toEqual({ + medianTimeToMergeHours: null, + realContributionPct: null, + queueHealthLevel: "low", + mergedSampleSize: 0, + assessedSampleSize: 0, + }); + }); + + it("computes the odd-count median time-to-merge in whole hours", () => { + const quality = buildPublicRepoQuality( + [ + merged("2026-06-01T00:00:00Z", "2026-06-01T02:00:00Z"), // 2h + merged("2026-06-01T00:00:00Z", "2026-06-01T06:00:00Z"), // 6h + merged("2026-06-01T00:00:00Z", "2026-06-01T10:00:00Z"), // 10h + ], + NOW, + ); + expect(quality.medianTimeToMergeHours).toBe(6); + expect(quality.mergedSampleSize).toBe(3); + }); + + it("averages the two middle values for an even-count median", () => { + expect( + buildPublicRepoQuality( + [merged("2026-06-01T00:00:00Z", "2026-06-01T02:00:00Z"), merged("2026-06-01T00:00:00Z", "2026-06-01T08:00:00Z")], + NOW, + ).medianTimeToMergeHours, + ).toBe(5); + }); + + it("excludes merges with missing or impossible timestamps from the median", () => { + const quality = buildPublicRepoQuality( + [ + merged("2026-06-01T00:00:00Z", "2026-06-01T04:00:00Z"), // 4h, valid + pr({ state: "merged", mergedAt: "2026-06-02T00:00:00Z" }), // no createdAt + pr({ state: "merged", createdAt: "2026-06-03T05:00:00Z", mergedAt: "2026-06-03T00:00:00Z" }), // merged < created + ], + NOW, + ); + expect(quality.medianTimeToMergeHours).toBe(4); + expect(quality.mergedSampleSize).toBe(3); + }); + + it("treats state=merged without mergedAt as merged for sample size", () => { + expect(buildPublicRepoQuality([pr({ state: "MERGED" })], NOW).mergedSampleSize).toBe(1); + }); + + it("computes the non-slop contribution share only over assessed merges", () => { + const quality = buildPublicRepoQuality( + [ + merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z", "clean"), + merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z", "LOW"), // case-insensitive non-slop + merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z", "high"), // slop + merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z"), // not assessed → excluded + ], + NOW, + ); + expect(quality.assessedSampleSize).toBe(3); + expect(quality.realContributionPct).toBe(67); // 2 of 3 assessed are non-slop + }); + + it("returns null real-contribution share when no merge is assessed", () => { + expect(buildPublicRepoQuality([merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z")], NOW).realContributionPct).toBeNull(); + }); + + it("classifies queue health by open volume and staleness", () => { + const openFresh = (count: number) => + Array.from({ length: count }, (_, i) => pr({ number: i + 1, state: "open", updatedAt: "2026-06-14T00:00:00Z" })); + const openStale = (count: number) => + Array.from({ length: count }, (_, i) => pr({ number: i + 100, state: "open", updatedAt: "2026-05-01T00:00:00Z" })); + + expect(buildPublicRepoQuality(openFresh(3), NOW).queueHealthLevel).toBe("low"); + expect(buildPublicRepoQuality(openFresh(6), NOW).queueHealthLevel).toBe("medium"); + expect(buildPublicRepoQuality(openStale(2), NOW).queueHealthLevel).toBe("medium"); // staleness path + expect(buildPublicRepoQuality(openFresh(20), NOW).queueHealthLevel).toBe("high"); + expect(buildPublicRepoQuality(openStale(8), NOW).queueHealthLevel).toBe("high"); + expect(buildPublicRepoQuality(openFresh(50), NOW).queueHealthLevel).toBe("critical"); + expect(buildPublicRepoQuality(openStale(20), NOW).queueHealthLevel).toBe("critical"); + }); + + it("falls back to createdAt, then ignores, when open PRs lack updatedAt", () => { + const staleByCreated = pr({ number: 1, state: "open", createdAt: "2026-05-01T00:00:00Z" }); // no updatedAt → use createdAt → stale + const alsoStale = pr({ number: 2, state: "open", createdAt: "2026-05-01T00:00:00Z" }); + const noTimestamps = pr({ number: 3, state: "open" }); // neither field → not counted as stale + expect(buildPublicRepoQuality([staleByCreated, alsoStale, noTimestamps], NOW).queueHealthLevel).toBe("medium"); + }); + + it("never exposes contributor-level or private terms", () => { + const quality = buildPublicRepoQuality([merged("2026-06-01T00:00:00Z", "2026-06-01T01:00:00Z", "clean")], NOW); + expect(JSON.stringify(quality)).not.toMatch(/wallet|hotkey|trust|reward|login|author|scoreability/i); + }); +}); From 842e3b9f071159c73051dc77dd39cc24b12886df Mon Sep 17 00:00:00 2001 From: bittoby <218712309+bittoby@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:01:10 +0000 Subject: [PATCH 2/4] feat(gate): #554 false-positive telemetry for hard-blocked PRs (measurement-only) --- migrations/0037_gate_outcomes.sql | 16 ++++ src/api/routes.ts | 10 +++ src/db/repositories.ts | 69 ++++++++++++++- src/db/schema.ts | 20 ++++- src/queue/processors.ts | 7 ++ src/services/gate-telemetry.ts | 98 +++++++++++++++++++++ src/types.ts | 31 +++++++ test/integration/api.test.ts | 27 ++++++ test/unit/gate-telemetry.test.ts | 138 ++++++++++++++++++++++++++++++ 9 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 migrations/0037_gate_outcomes.sql create mode 100644 src/services/gate-telemetry.ts create mode 100644 test/unit/gate-telemetry.test.ts diff --git a/migrations/0037_gate_outcomes.sql b/migrations/0037_gate_outcomes.sql new file mode 100644 index 0000000000..b1d8fc9caf --- /dev/null +++ b/migrations/0037_gate_outcomes.sql @@ -0,0 +1,16 @@ +-- #554: gate false-positive telemetry. One row per (repo, PR) capturing the latest gate HARD-BLOCK, later +-- correlated with an eventual merge/override (resolution) to measure each gate type's false-positive rate. +-- No PII: only repo, PR number, gate pack, blocker codes, and timestamps. +CREATE TABLE gate_outcomes ( + repo_full_name TEXT NOT NULL, + pr_number INTEGER NOT NULL, + gate_pack TEXT NOT NULL DEFAULT 'gittensor', + blocker_codes_json TEXT NOT NULL DEFAULT '[]', + blocked_at TEXT NOT NULL, + resolution TEXT, + resolved_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (repo_full_name, pr_number) +); +CREATE INDEX gate_outcomes_resolution_idx ON gate_outcomes (resolution); diff --git a/src/api/routes.ts b/src/api/routes.ts index d89041e58b..fd419c8d4c 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -73,6 +73,7 @@ import { summarizeRepoSyncOpenPullRequests, listSignalSnapshots, listPullRequests, + listGateOutcomes, listRepositories, getLatestUpstreamRulesetSnapshot, listUpstreamDriftReports, @@ -156,6 +157,7 @@ import { import { buildOperatorDashboardPayload } from "../services/operator-dashboard"; import { buildSelfDogfoodRegistrationPack, resolveSelfDogfoodRepoFullName } from "../services/self-dogfood-registration-pack"; import { buildSubnetInterfaceDescriptor } from "../services/subnet-interface"; +import { buildGateFalsePositiveReport } from "../services/gate-telemetry"; import { buildWeeklyValueReport, formatWeeklyValueReportMarkdown, @@ -2715,6 +2717,14 @@ export function createApp() { ); }); + // Gate false-positive telemetry (#554). Internal/maintainer-authenticated; never public. Returns the + // per-gate-type false-positive rate (blocked-then-merged/overridden) for a repo so maintainers can decide + // whether to move a gate from advisory to block. No PII or reward/trust fields. + app.get("/v1/internal/repos/:owner/:repo/gate-telemetry", async (c) => { + const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`; + return c.json(buildGateFalsePositiveReport(await listGateOutcomes(c.env, fullName), fullName)); + }); + // Maintainer BYOK provider key. GET returns secret-free status only; POST stores it encrypted at rest; // DELETE removes it. The plaintext key is never logged and never returned. app.get("/v1/internal/repos/:owner/:repo/ai-key", async (c) => { diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5d94002c85..f72ea4cdfb 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, gte, inArray, not, or, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, gte, inArray, isNull, not, or, sql, type SQL } from "drizzle-orm"; import { getDb } from "./client"; import { advisories, @@ -47,6 +47,7 @@ import { repoSyncState, repositoryAiKeys, repositorySettings, + gateOutcomes, scorePreviews, scoringModelSnapshots, signalSnapshots, @@ -134,6 +135,8 @@ import type { RepoSyncStateRecord, RepositorySettings, RepositoryRecord, + GateOutcomeRecord, + GateOutcomeResolution, ScorePreviewRecord, ScoringModelSnapshotRecord, SignalSnapshotRecord, @@ -5000,3 +5003,67 @@ function extractLinkedPrNumbers(text: string): number[] { const matches = [...text.matchAll(/\b(?:PR|pull request)\s+#(\d+)\b/gi)]; return [...new Set(matches.map((match) => Number(match[1])).filter((value) => Number.isInteger(value) && value > 0))]; } + +// ── Gate false-positive telemetry (#554) ─────────────────────────────────────────────────────────── + +/** Record (or refresh) a gate HARD-BLOCK for a PR. Re-blocking clears any prior resolution so a block + * that fires again is not still counted as a resolved false positive. */ +export async function recordGateBlockOutcome( + env: Env, + outcome: { repoFullName: string; prNumber: number; gatePack: string; blockerCodes: string[] }, +): Promise { + const db = getDb(env.DB); + const now = nowIso(); + const blockerCodesJson = jsonString([...new Set(outcome.blockerCodes.filter((code) => typeof code === "string" && code.length > 0))]); + await db + .insert(gateOutcomes) + .values({ + repoFullName: outcome.repoFullName, + prNumber: outcome.prNumber, + gatePack: outcome.gatePack, + blockerCodesJson, + blockedAt: now, + resolution: null, + resolvedAt: null, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [gateOutcomes.repoFullName, gateOutcomes.prNumber], + set: { gatePack: outcome.gatePack, blockerCodesJson, blockedAt: now, resolution: null, resolvedAt: null, updatedAt: now }, + }); +} + +/** Mark a previously blocked PR as a false positive (merged or overridden). No-op unless an unresolved + * block row exists, so a plain close — or a PR that was never blocked — is never counted. */ +export async function resolveGateOutcome(env: Env, repoFullName: string, prNumber: number, resolution: GateOutcomeResolution): Promise { + const db = getDb(env.DB); + await db + .update(gateOutcomes) + .set({ resolution, resolvedAt: nowIso(), updatedAt: nowIso() }) + .where(and(eq(gateOutcomes.repoFullName, repoFullName), eq(gateOutcomes.prNumber, prNumber), isNull(gateOutcomes.resolution))); +} + +export async function listGateOutcomes(env: Env, repoFullName?: string): Promise { + const db = getDb(env.DB); + const rows = repoFullName + ? await db.select().from(gateOutcomes).where(eq(gateOutcomes.repoFullName, repoFullName)).limit(1000) + : await db.select().from(gateOutcomes).limit(2000); + return rows.map(toGateOutcomeRecord); +} + +function toGateOutcomeRecord(row: typeof gateOutcomes.$inferSelect): GateOutcomeRecord { + return { + repoFullName: row.repoFullName, + prNumber: row.prNumber, + gatePack: row.gatePack, + blockerCodes: parseGateBlockerCodes(row.blockerCodesJson), + blockedAt: row.blockedAt, + resolution: (row.resolution as GateOutcomeResolution | null) ?? null, + resolvedAt: row.resolvedAt ?? null, + }; +} + +function parseGateBlockerCodes(raw: string): string[] { + const parsed = parseJson(raw, [] as unknown); + return Array.isArray(parsed) ? parsed.filter((code): code is string => typeof code === "string" && code.length > 0) : []; +} diff --git a/src/db/schema.ts b/src/db/schema.ts index 039db757bb..d99106815b 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,4 @@ -import { index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; +import { index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; // Timestamp columns use a drizzle $defaultFn so an insert that omits the column gets a real ISO-8601 // timestamp. A static `.default("CURRENT_TIMESTAMP")` would make drizzle inject the literal STRING // "CURRENT_TIMESTAMP" (it applies static defaults client-side, never reaching SQLite's CURRENT_TIMESTAMP), @@ -91,6 +91,24 @@ export const repositoryAiKeys = sqliteTable("repository_ai_keys", { updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()), }); +// Gate false-positive telemetry (#554): the latest hard-block per (repo, PR), correlated with an eventual +// merge/override so the maintainer dashboard can show each gate type's false-positive rate. No PII. +export const gateOutcomes = sqliteTable( + "gate_outcomes", + { + repoFullName: text("repo_full_name").notNull(), + prNumber: integer("pr_number").notNull(), + gatePack: text("gate_pack").notNull().default("gittensor"), + blockerCodesJson: text("blocker_codes_json").notNull().default("[]"), + blockedAt: text("blocked_at").notNull(), + resolution: text("resolution"), + resolvedAt: text("resolved_at"), + createdAt: text("created_at").notNull().$defaultFn(() => nowIso()), + updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()), + }, + (table) => [primaryKey({ columns: [table.repoFullName, table.prNumber] }), index("gate_outcomes_resolution_idx").on(table.resolution)], +); + export const repoSyncState = sqliteTable("repo_sync_state", { repoFullName: text("repo_full_name").primaryKey(), status: text("status").notNull().default("never_synced"), diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 6cd4685e51..6bd74098bb 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -85,6 +85,7 @@ import { detectNotificationEvents } from "../notifications/events"; import { deliverNotification, detectIssueWatchEvents, evaluateNotificationEvent } from "../notifications/service"; import { getOrCreateScoringModelSnapshot, refreshScoringModelSnapshot } from "../scoring/model"; import { buildAndPersistContributorDecisionPack, loadDecisionPackSharedInputs } from "../services/decision-pack"; +import { recordGateOutcomeForEvaluation, resolveMergedGateOutcome } from "../services/gate-telemetry"; import { buildContributorEvidenceGraph, CONTRIBUTOR_EVIDENCE_GRAPH_SIGNAL, @@ -729,6 +730,9 @@ async function processGitHubWebhook(env: Env, deliveryId: string, eventName: str if (payload.repository?.full_name && payload.pull_request) { const repoFullName = payload.repository.full_name; const pr = await upsertPullRequestFromGitHub(env, repoFullName, payload.pull_request); + // Gate false-positive telemetry (#554): a previously gate-blocked PR that is now merged is a false + // positive. No-op for non-merge events or PRs that were never blocked. + await resolveMergedGateOutcome(env, repoFullName, pr.number, payload.action, payload.pull_request); const [repo, settings, otherOpenPullRequests] = await Promise.all([ getRepository(env, repoFullName), resolveRepositorySettings(env, repoFullName), @@ -1223,6 +1227,9 @@ async function maybePublishPrPublicSurface( if (gateCheckResult?.kind === "permission_missing") { await auditGateCheckPermissionMissing(env, author, repoFullName, pr.number, webhook.deliveryId, gateCheckResult.warning); } + // Gate false-positive telemetry (#554): record a hard block so an eventual merge can mark it a false + // positive. No-op unless the gate concluded `failure` (confirmed-contributor hard block). + await recordGateOutcomeForEvaluation(env, { repoFullName, prNumber: pr.number, gatePack: settings.gatePack, evaluation: gateEvaluation }); } } catch (error) { // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral diff --git a/src/services/gate-telemetry.ts b/src/services/gate-telemetry.ts new file mode 100644 index 0000000000..b4bd07b60d --- /dev/null +++ b/src/services/gate-telemetry.ts @@ -0,0 +1,98 @@ +import type { GateCheckEvaluation } from "../rules/advisory"; +import { recordGateBlockOutcome, resolveGateOutcome } from "../db/repositories"; +import type { GateFalsePositiveRate, GateFalsePositiveReport, GateOutcomeRecord, GitHubPullRequestPayload } from "../types"; + +// Gate false-positive telemetry (#554). Maintainers won't move a gate from advisory to block without +// evidence it is precise, so we record every hard-block and correlate it with the PR's eventual merge or +// override to expose a per-gate-type false-positive rate. Pure aggregation + thin, branch-light recording +// helpers (the branching lives here so the deep webhook processor stays a straight-line call site). + +/** + * Record a gate hard-block (conclusion `failure`) for later false-positive correlation. No-op for any + * non-blocking outcome — only confirmed-contributor hard blocks reach `failure`, so advisory/neutral runs + * are never counted. + */ +export async function recordGateOutcomeForEvaluation( + env: Env, + args: { repoFullName: string; prNumber: number; gatePack: string; evaluation: GateCheckEvaluation | undefined }, +): Promise { + if (!args.evaluation || args.evaluation.conclusion !== "failure") return; + // Best-effort telemetry: a write failure must never disrupt gate/webhook processing. + try { + await recordGateBlockOutcome(env, { + repoFullName: args.repoFullName, + prNumber: args.prNumber, + gatePack: args.gatePack, + blockerCodes: args.evaluation.blockers.map((blocker) => blocker.code), + }); + } catch { + return; + } +} + +/** + * A previously gate-blocked PR that is later merged is a false positive — the block did not reflect a real + * defect. Resolves only on merge; a plain close is a true positive (the block held), and PRs that were + * never blocked are unaffected (the DB update only touches an existing unresolved row). + */ +export async function resolveMergedGateOutcome( + env: Env, + repoFullName: string, + prNumber: number, + action: string | undefined, + pullRequest: Pick, +): Promise { + if (action !== "closed" || !pullRequest.merged_at) return; + // Best-effort telemetry: a write failure must never disrupt gate/webhook processing. + try { + await resolveGateOutcome(env, repoFullName, prNumber, "merged"); + } catch { + return; + } +} + +/** + * Aggregate a false-positive rate overall and per gate type (blocker code). A "false positive" is any + * recorded block whose outcome was later resolved (merged or overridden). + */ +export function buildGateFalsePositiveReport( + outcomes: GateOutcomeRecord[], + repoFullName: string | null = null, +): GateFalsePositiveReport { + const byCode = new Map(); + let totalBlocked = 0; + let totalFalsePositives = 0; + + for (const outcome of outcomes) { + totalBlocked += 1; + const falsePositive = outcome.resolution != null; + if (falsePositive) totalFalsePositives += 1; + for (const code of new Set(outcome.blockerCodes)) { + const entry = byCode.get(code) ?? { blocked: 0, falsePositives: 0 }; + entry.blocked += 1; + if (falsePositive) entry.falsePositives += 1; + byCode.set(code, entry); + } + } + + const byGateType: GateFalsePositiveRate[] = [...byCode.entries()] + .map(([code, entry]) => ({ + code, + blocked: entry.blocked, + falsePositives: entry.falsePositives, + falsePositiveRate: rate(entry.falsePositives, entry.blocked), + })) + .sort((left, right) => left.code.localeCompare(right.code)); + + return { + repoFullName, + totalBlocked, + totalFalsePositives, + falsePositiveRate: rate(totalFalsePositives, totalBlocked), + byGateType, + }; +} + +function rate(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : Math.round((numerator / denominator) * 1000) / 1000; +} diff --git a/src/types.ts b/src/types.ts index f16a5d1ca6..cae2d10079 100644 --- a/src/types.ts +++ b/src/types.ts @@ -449,6 +449,37 @@ export type RepositorySettings = { updatedAt?: string | null | undefined; }; +// Gate false-positive telemetry (#554). A blocked PR that is later merged or overridden is a false +// positive — the hard block did not reflect a real defect. +export type GateOutcomeResolution = "merged" | "overridden"; + +export type GateOutcomeRecord = { + repoFullName: string; + prNumber: number; + gatePack: string; + blockerCodes: string[]; + blockedAt: string; + resolution?: GateOutcomeResolution | null | undefined; + resolvedAt?: string | null | undefined; +}; + +export type GateFalsePositiveRate = { + /** Gate blocker code (the gate "type"), e.g. `missing_linked_issue`, `duplicate_pr_risk`. */ + code: string; + blocked: number; + falsePositives: number; + /** falsePositives / blocked, in [0, 1], rounded to 3 dp. */ + falsePositiveRate: number; +}; + +export type GateFalsePositiveReport = { + repoFullName: string | null; + totalBlocked: number; + totalFalsePositives: number; + falsePositiveRate: number; + byGateType: GateFalsePositiveRate[]; +}; + export type CommandAuthorizationRole = "maintainer" | "collaborator" | "pr_author" | "confirmed_miner"; export type RepositoryCommandAuthorizationPolicy = { diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index faad53c53e..16acaf5f90 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -33,6 +33,8 @@ import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome, + recordGateBlockOutcome, + resolveGateOutcome, } from "../../src/db/repositories"; import { createApp } from "../../src/api/routes"; import { clearPublicRepoStatsCacheForTests } from "../../src/github/public"; @@ -193,6 +195,31 @@ describe("api routes", () => { await expect(unavailable.json()).resolves.toMatchObject({ error: "github_repo_stats_unavailable" }); }); + it("exposes per-gate-type false-positive telemetry on the internal endpoint (#554)", async () => { + const app = createApp(); + const env = createTestEnv(); + + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", blockerCodes: ["missing_linked_issue"] }); + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 2, gatePack: "gittensor", blockerCodes: ["missing_linked_issue", "duplicate_pr_risk"] }); + await resolveGateOutcome(env, "acme/widgets", 1, "merged"); // blocked-then-merged → false positive + + const unauthorized = await app.request("/v1/internal/repos/acme/widgets/gate-telemetry", {}, env); + expect(unauthorized.status).toBe(401); + + const response = await app.request("/v1/internal/repos/acme/widgets/gate-telemetry", { headers: internalHeaders(env) }, env); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + repoFullName: "acme/widgets", + totalBlocked: 2, + totalFalsePositives: 1, + falsePositiveRate: 0.5, + byGateType: expect.arrayContaining([ + { code: "missing_linked_issue", blocked: 2, falsePositives: 1, falsePositiveRate: 0.5 }, + { code: "duplicate_pr_risk", blocked: 1, falsePositives: 0, falsePositiveRate: 0 }, + ]), + }); + }); + it("rejects invalid public GitHub repo stats paths before calling GitHub", async () => { const app = createApp(); const env = createTestEnv(); diff --git a/test/unit/gate-telemetry.test.ts b/test/unit/gate-telemetry.test.ts new file mode 100644 index 0000000000..3692390c6a --- /dev/null +++ b/test/unit/gate-telemetry.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { + buildGateFalsePositiveReport, + recordGateOutcomeForEvaluation, + resolveMergedGateOutcome, +} from "../../src/services/gate-telemetry"; +import { listGateOutcomes, recordGateBlockOutcome, resolveGateOutcome } from "../../src/db/repositories"; +import { createTestEnv } from "../helpers/d1"; +import type { GateCheckEvaluation } from "../../src/rules/advisory"; +import type { GateOutcomeRecord, GateOutcomeResolution } from "../../src/types"; + +function outcome(blockerCodes: string[], resolution: GateOutcomeResolution | null = null, prNumber = 1): GateOutcomeRecord { + return { + repoFullName: "acme/widgets", + prNumber, + gatePack: "gittensor", + blockerCodes, + blockedAt: "2026-06-15T00:00:00.000Z", + resolution, + resolvedAt: resolution ? "2026-06-16T00:00:00.000Z" : null, + }; +} + +function failureEval(codes: string[]): GateCheckEvaluation { + return { + enabled: true, + conclusion: "failure", + title: "Gittensory Gate: blocked", + summary: "blocked", + blockers: codes.map((code) => ({ code, title: code, severity: "warning", detail: "d" })), + warnings: [], + }; +} + +describe("buildGateFalsePositiveReport", () => { + it("returns zeros for no recorded outcomes", () => { + expect(buildGateFalsePositiveReport([])).toEqual({ + repoFullName: null, + totalBlocked: 0, + totalFalsePositives: 0, + falsePositiveRate: 0, + byGateType: [], + }); + }); + + it("aggregates overall and per-gate-type false-positive rates", () => { + const report = buildGateFalsePositiveReport( + [ + outcome(["missing_linked_issue", "duplicate_pr_risk"], null, 1), + outcome(["missing_linked_issue"], "merged", 2), + outcome(["duplicate_pr_risk", "slop_gate"], "overridden", 3), + ], + "acme/widgets", + ); + expect(report).toEqual({ + repoFullName: "acme/widgets", + totalBlocked: 3, + totalFalsePositives: 2, + falsePositiveRate: 0.667, + byGateType: [ + { code: "duplicate_pr_risk", blocked: 2, falsePositives: 1, falsePositiveRate: 0.5 }, + { code: "missing_linked_issue", blocked: 2, falsePositives: 1, falsePositiveRate: 0.5 }, + { code: "slop_gate", blocked: 1, falsePositives: 1, falsePositiveRate: 1 }, + ], + }); + }); + + it("counts a blocker code at most once per outcome", () => { + const report = buildGateFalsePositiveReport([outcome(["dup", "dup"], "merged")]); + expect(report.byGateType).toEqual([{ code: "dup", blocked: 1, falsePositives: 1, falsePositiveRate: 1 }]); + }); +}); + +describe("gate outcome recording", () => { + it("records a hard block from a failure evaluation and ignores non-blocking outcomes", async () => { + const env = createTestEnv(); + await recordGateOutcomeForEvaluation(env, { repoFullName: "acme/widgets", prNumber: 7, gatePack: "gittensor", evaluation: failureEval(["duplicate_pr_risk"]) }); + await recordGateOutcomeForEvaluation(env, { repoFullName: "acme/widgets", prNumber: 8, gatePack: "gittensor", evaluation: { ...failureEval([]), conclusion: "success" } }); + await recordGateOutcomeForEvaluation(env, { repoFullName: "acme/widgets", prNumber: 9, gatePack: "gittensor", evaluation: undefined }); + + const outcomes = await listGateOutcomes(env, "acme/widgets"); + expect(outcomes).toHaveLength(1); + expect(outcomes[0]).toMatchObject({ prNumber: 7, gatePack: "gittensor", blockerCodes: ["duplicate_pr_risk"], resolution: null }); + }); + + it("marks a blocked-then-merged PR as a false positive but leaves plain closes alone", async () => { + const env = createTestEnv(); + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", blockerCodes: ["slop_gate"] }); + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 2, gatePack: "gittensor", blockerCodes: ["slop_gate"] }); + + await resolveMergedGateOutcome(env, "acme/widgets", 1, "closed", { merged_at: "2026-06-16T00:00:00.000Z" }); + await resolveMergedGateOutcome(env, "acme/widgets", 2, "closed", { merged_at: null }); // closed unmerged → not a false positive + await resolveMergedGateOutcome(env, "acme/widgets", 1, "synchronize", { merged_at: "x" }); // non-close → no-op + + const report = buildGateFalsePositiveReport(await listGateOutcomes(env, "acme/widgets"), "acme/widgets"); + expect(report.totalBlocked).toBe(2); + expect(report.totalFalsePositives).toBe(1); + }); + + it("re-blocking clears a prior resolution and resolve only touches unresolved rows", async () => { + const env = createTestEnv(); + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", blockerCodes: ["a"] }); + await resolveGateOutcome(env, "acme/widgets", 1, "merged"); + expect((await listGateOutcomes(env, "acme/widgets"))[0]?.resolution).toBe("merged"); + + // Re-block clears the resolution; a subsequent resolve on a never-blocked PR is a no-op. + await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", blockerCodes: ["a", "b"] }); + await resolveGateOutcome(env, "acme/widgets", 999, "merged"); + const rows = await listGateOutcomes(env, "acme/widgets"); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ resolution: null, blockerCodes: ["a", "b"] }); + }); + + it("swallows write failures so telemetry never disrupts gate/webhook processing", async () => { + const brokenEnv = { + DB: { + prepare() { + throw new Error("db unavailable"); + }, + }, + } as unknown as Env; + + await expect( + recordGateOutcomeForEvaluation(brokenEnv, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", evaluation: failureEval(["slop_gate"]) }), + ).resolves.toBeUndefined(); + await expect( + resolveMergedGateOutcome(brokenEnv, "acme/widgets", 1, "closed", { merged_at: "2026-06-16T00:00:00.000Z" }), + ).resolves.toBeUndefined(); + }); + + it("lists outcomes scoped to a repo or across all repos", async () => { + const env = createTestEnv(); + await recordGateBlockOutcome(env, { repoFullName: "acme/a", prNumber: 1, gatePack: "gittensor", blockerCodes: ["x"] }); + await recordGateBlockOutcome(env, { repoFullName: "acme/b", prNumber: 1, gatePack: "gittensor", blockerCodes: ["y"] }); + expect(await listGateOutcomes(env, "acme/a")).toHaveLength(1); + expect(await listGateOutcomes(env)).toHaveLength(2); + }); +}); From a514e56f5947e66abcaa973683ff151447167f38 Mon Sep 17 00:00:00 2001 From: bittoby <218712309+bittoby@users.noreply.github.com> Date: Tue, 16 Jun 2026 21:47:40 +0000 Subject: [PATCH 3/4] fix ci --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index bff2cecce7..d1a1c46bca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9029,9 +9029,9 @@ } }, "node_modules/hono": { - "version": "4.12.23", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.23.tgz", - "integrity": "sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==", + "version": "4.12.25", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.25.tgz", + "integrity": "sha512-2NFaIyNVgJmBs/ecmtGzlmluTFs5cHEWGTdu0t1HBwYzoGXOL5nUQBRMXsXWla5i4KkG//QMzVP88m1+I3fdAQ==", "license": "MIT", "engines": { "node": ">=16.9.0" From e0193aa65c0031df5d71fe4e8ddbeda6227eb172 Mon Sep 17 00:00:00 2001 From: bittoby <218712309+bittoby@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:30:55 +0000 Subject: [PATCH 4/4] update migration name --- migrations/{0040_gate_outcomes.sql => 0041_gate_outcomes.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migrations/{0040_gate_outcomes.sql => 0041_gate_outcomes.sql} (100%) diff --git a/migrations/0040_gate_outcomes.sql b/migrations/0041_gate_outcomes.sql similarity index 100% rename from migrations/0040_gate_outcomes.sql rename to migrations/0041_gate_outcomes.sql