feat(gate): #554 false-positive telemetry for hard-blocked PRs (measurement-only)

migrations/0041_gate_outcomes.sql

@@ -0,0 +1,16 @@
+-- #554: gate false-positive telemetry. One row per (repo, PR) capturing the latest gate HARD-BLOCK, later
+-- correlated with an eventual merge/override (resolution) to measure each gate type's false-positive rate.
+-- No PII: only repo, PR number, gate pack, blocker codes, and timestamps.
+CREATE TABLE gate_outcomes (
+  repo_full_name TEXT NOT NULL,
+  pr_number INTEGER NOT NULL,
+  gate_pack TEXT NOT NULL DEFAULT 'gittensor',
+  blocker_codes_json TEXT NOT NULL DEFAULT '[]',
+  blocked_at TEXT NOT NULL,
+  resolution TEXT,
+  resolved_at TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (repo_full_name, pr_number)
+);
+CREATE INDEX gate_outcomes_resolution_idx ON gate_outcomes (resolution);

src/api/routes.ts

@@ -1,4 +1,4 @@
 import { Hono, type Context } from "hono";
 import { z } from "zod";
 import { analyzePRQueue, type AuthorRole, type ChecksStatus } from "../queue-intelligence";
 import { completeGitHubWebOAuth, createSessionFromGitHubToken, pollGitHubDeviceFlow, startGitHubDeviceFlow, startGitHubWebOAuth } from "../auth/github-oauth";
@@ -73,6 +73,7 @@
   summarizeRepoSyncOpenPullRequests,
   listSignalSnapshots,
   listPullRequests,
+  listGateOutcomes,
   listRepositories,
   getLatestUpstreamRulesetSnapshot,
   listUpstreamDriftReports,
@@ -158,6 +159,7 @@
 import { buildOperatorDashboardPayload } from "../services/operator-dashboard";
 import { buildSelfDogfoodRegistrationPack, resolveSelfDogfoodRepoFullName } from "../services/self-dogfood-registration-pack";
 import { buildSubnetInterfaceDescriptor } from "../services/subnet-interface";
+import { buildGateFalsePositiveReport } from "../services/gate-telemetry";
 import { buildPublicRepoQuality, type PublicRepoQuality } from "../services/public-repo-quality";
 import { buildShieldsBadge, renderBadgeSvg, renderUnavailableBadgeSvg } from "./badge";
 import {
@@ -2927,6 +2929,14 @@
     );
   });
 
+  // Gate false-positive telemetry (#554). Internal/maintainer-authenticated; never public. Returns the
+  // per-gate-type false-positive rate (blocked-then-merged/overridden) for a repo so maintainers can decide
+  // whether to move a gate from advisory to block. No PII or reward/trust fields.
+  app.get("/v1/internal/repos/:owner/:repo/gate-telemetry", async (c) => {
+    const fullName = `${c.req.param("owner")}/${c.req.param("repo")}`;
+    return c.json(buildGateFalsePositiveReport(await listGateOutcomes(c.env, fullName), fullName));
+  });
+
   // Maintainer BYOK provider key. GET returns secret-free status only; POST stores it encrypted at rest;
   // DELETE removes it. The plaintext key is never logged and never returned.
   app.get("/v1/internal/repos/:owner/:repo/ai-key", async (c) => {

src/db/repositories.ts

@@ -1,4 +1,4 @@
-import { and, desc, eq, gte, inArray, not, or, sql, type SQL } from "drizzle-orm";
+import { and, desc, eq, gte, inArray, isNull, not, or, sql, type SQL } from "drizzle-orm";
 import { getDb } from "./client";
 import {
   advisories,
@@ -47,6 +47,7 @@
   repoSyncState,
   repositoryAiKeys,
   repositorySettings,
+  gateOutcomes,
   scorePreviews,
   scoringModelSnapshots,
   signalSnapshots,
@@ -134,6 +135,8 @@
   RepoSyncStateRecord,
   RepositorySettings,
   RepositoryRecord,
+  GateOutcomeRecord,
+  GateOutcomeResolution,
   ScorePreviewRecord,
   ScoringModelSnapshotRecord,
   SignalSnapshotRecord,
@@ -5022,3 +5025,67 @@
   const matches = [...text.matchAll(/\b(?:PR|pull request)\s+#(\d+)\b/gi)];
   return [...new Set(matches.map((match) => Number(match[1])).filter((value) => Number.isInteger(value) && value > 0))];
 }
+
+// ── Gate false-positive telemetry (#554) ───────────────────────────────────────────────────────────
+
+/** Record (or refresh) a gate HARD-BLOCK for a PR. Re-blocking clears any prior resolution so a block
+ *  that fires again is not still counted as a resolved false positive. */
+export async function recordGateBlockOutcome(
+  env: Env,
+  outcome: { repoFullName: string; prNumber: number; gatePack: string; blockerCodes: string[] },
+): Promise<void> {
+  const db = getDb(env.DB);
+  const now = nowIso();
+  const blockerCodesJson = jsonString([...new Set(outcome.blockerCodes.filter((code) => typeof code === "string" && code.length > 0))]);
+  await db
+    .insert(gateOutcomes)
+    .values({
+      repoFullName: outcome.repoFullName,
+      prNumber: outcome.prNumber,
+      gatePack: outcome.gatePack,
+      blockerCodesJson,
+      blockedAt: now,
+      resolution: null,
+      resolvedAt: null,
+      updatedAt: now,
+    })
+    .onConflictDoUpdate({
+      target: [gateOutcomes.repoFullName, gateOutcomes.prNumber],
+      set: { gatePack: outcome.gatePack, blockerCodesJson, blockedAt: now, resolution: null, resolvedAt: null, updatedAt: now },
+    });
+}
+
+/** Mark a previously blocked PR as a false positive (merged or overridden). No-op unless an unresolved
+ *  block row exists, so a plain close — or a PR that was never blocked — is never counted. */
+export async function resolveGateOutcome(env: Env, repoFullName: string, prNumber: number, resolution: GateOutcomeResolution): Promise<void> {
+  const db = getDb(env.DB);
+  await db
+    .update(gateOutcomes)
+    .set({ resolution, resolvedAt: nowIso(), updatedAt: nowIso() })
+    .where(and(eq(gateOutcomes.repoFullName, repoFullName), eq(gateOutcomes.prNumber, prNumber), isNull(gateOutcomes.resolution)));
+}
+
+export async function listGateOutcomes(env: Env, repoFullName?: string): Promise<GateOutcomeRecord[]> {
+  const db = getDb(env.DB);
+  const rows = repoFullName
+    ? await db.select().from(gateOutcomes).where(eq(gateOutcomes.repoFullName, repoFullName)).limit(1000)
+    : await db.select().from(gateOutcomes).limit(2000);
+  return rows.map(toGateOutcomeRecord);
+}
+
+function toGateOutcomeRecord(row: typeof gateOutcomes.$inferSelect): GateOutcomeRecord {
+  return {
+    repoFullName: row.repoFullName,
+    prNumber: row.prNumber,
+    gatePack: row.gatePack,
+    blockerCodes: parseGateBlockerCodes(row.blockerCodesJson),
+    blockedAt: row.blockedAt,
+    resolution: (row.resolution as GateOutcomeResolution | null) ?? null,
+    resolvedAt: row.resolvedAt ?? null,
+  };
+}
+
+function parseGateBlockerCodes(raw: string): string[] {
+  const parsed = parseJson(raw, [] as unknown);
+  return Array.isArray(parsed) ? parsed.filter((code): code is string => typeof code === "string" && code.length > 0) : [];
+}

src/db/schema.ts

@@ -1,4 +1,4 @@
-import { index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
+import { index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
 // Timestamp columns use a drizzle $defaultFn so an insert that omits the column gets a real ISO-8601
 // timestamp. A static `.default("CURRENT_TIMESTAMP")` would make drizzle inject the literal STRING
 // "CURRENT_TIMESTAMP" (it applies static defaults client-side, never reaching SQLite's CURRENT_TIMESTAMP),
@@ -95,6 +95,24 @@
   updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()),
 });
 
+// Gate false-positive telemetry (#554): the latest hard-block per (repo, PR), correlated with an eventual
+// merge/override so the maintainer dashboard can show each gate type's false-positive rate. No PII.
+export const gateOutcomes = sqliteTable(
+  "gate_outcomes",
+  {
+    repoFullName: text("repo_full_name").notNull(),
+    prNumber: integer("pr_number").notNull(),
+    gatePack: text("gate_pack").notNull().default("gittensor"),
+    blockerCodesJson: text("blocker_codes_json").notNull().default("[]"),
+    blockedAt: text("blocked_at").notNull(),
+    resolution: text("resolution"),
+    resolvedAt: text("resolved_at"),
+    createdAt: text("created_at").notNull().$defaultFn(() => nowIso()),
+    updatedAt: text("updated_at").notNull().$defaultFn(() => nowIso()),
+  },
+  (table) => [primaryKey({ columns: [table.repoFullName, table.prNumber] }), index("gate_outcomes_resolution_idx").on(table.resolution)],
+);
+
 export const repoSyncState = sqliteTable("repo_sync_state", {
   repoFullName: text("repo_full_name").primaryKey(),
   status: text("status").notNull().default("never_synced"),

src/queue/processors.ts

@@ -1,4 +1,4 @@
 import {
   countOpenIssues,
   countOpenPullRequests,
   getAgentCommandAnswer,
@@ -86,6 +86,7 @@
 import { deliverNotification, detectIssueWatchEvents, evaluateNotificationEvent } from "../notifications/service";
 import { getOrCreateScoringModelSnapshot, refreshScoringModelSnapshot } from "../scoring/model";
 import { buildAndPersistContributorDecisionPack, loadDecisionPackSharedInputs } from "../services/decision-pack";
+import { recordGateOutcomeForEvaluation, resolveMergedGateOutcome } from "../services/gate-telemetry";
 import {
   buildContributorEvidenceGraph,
   CONTRIBUTOR_EVIDENCE_GRAPH_SIGNAL,
@@ -759,6 +760,9 @@
     if (payload.repository?.full_name && payload.pull_request) {
       const repoFullName = payload.repository.full_name;
       const pr = await upsertPullRequestFromGitHub(env, repoFullName, payload.pull_request);
+      // Gate false-positive telemetry (#554): a previously gate-blocked PR that is now merged is a false
+      // positive. No-op for non-merge events or PRs that were never blocked.
+      await resolveMergedGateOutcome(env, repoFullName, pr.number, payload.action, payload.pull_request);
       const [repo, settings, otherOpenPullRequests] = await Promise.all([
         getRepository(env, repoFullName),
         resolveRepositorySettings(env, repoFullName),
@@ -1315,6 +1319,9 @@
           gateFinalized = true;
         }
       }
+      // Gate false-positive telemetry (#554): record a hard block so an eventual merge can mark it a false
+      // positive. No-op unless the gate concluded `failure` (confirmed-contributor hard block).
+      await recordGateOutcomeForEvaluation(env, { repoFullName, prNumber: pr.number, gatePack: settings.gatePack, evaluation: gateEvaluation });
     }
   } catch (error) {
     // The pending Gate check was posted but evaluation could not finish. Finalize it to a neutral

src/services/gate-telemetry.ts

@@ -0,0 +1,98 @@
+import type { GateCheckEvaluation } from "../rules/advisory";
+import { recordGateBlockOutcome, resolveGateOutcome } from "../db/repositories";
+import type { GateFalsePositiveRate, GateFalsePositiveReport, GateOutcomeRecord, GitHubPullRequestPayload } from "../types";
+
+// Gate false-positive telemetry (#554). Maintainers won't move a gate from advisory to block without
+// evidence it is precise, so we record every hard-block and correlate it with the PR's eventual merge or
+// override to expose a per-gate-type false-positive rate. Pure aggregation + thin, branch-light recording
+// helpers (the branching lives here so the deep webhook processor stays a straight-line call site).
+
+/**
+ * Record a gate hard-block (conclusion `failure`) for later false-positive correlation. No-op for any
+ * non-blocking outcome — only confirmed-contributor hard blocks reach `failure`, so advisory/neutral runs
+ * are never counted.
+ */
+export async function recordGateOutcomeForEvaluation(
+  env: Env,
+  args: { repoFullName: string; prNumber: number; gatePack: string; evaluation: GateCheckEvaluation | undefined },
+): Promise<void> {
+  if (!args.evaluation || args.evaluation.conclusion !== "failure") return;
+  // Best-effort telemetry: a write failure must never disrupt gate/webhook processing.
+  try {
+    await recordGateBlockOutcome(env, {
+      repoFullName: args.repoFullName,
+      prNumber: args.prNumber,
+      gatePack: args.gatePack,
+      blockerCodes: args.evaluation.blockers.map((blocker) => blocker.code),
+    });
+  } catch {
+    return;
+  }
+}
+
+/**
+ * A previously gate-blocked PR that is later merged is a false positive — the block did not reflect a real
+ * defect. Resolves only on merge; a plain close is a true positive (the block held), and PRs that were
+ * never blocked are unaffected (the DB update only touches an existing unresolved row).
+ */
+export async function resolveMergedGateOutcome(
+  env: Env,
+  repoFullName: string,
+  prNumber: number,
+  action: string | undefined,
+  pullRequest: Pick<GitHubPullRequestPayload, "merged_at">,
+): Promise<void> {
+  if (action !== "closed" || !pullRequest.merged_at) return;
+  // Best-effort telemetry: a write failure must never disrupt gate/webhook processing.
+  try {
+    await resolveGateOutcome(env, repoFullName, prNumber, "merged");
+  } catch {
+    return;
+  }
+}
+
+/**
+ * Aggregate a false-positive rate overall and per gate type (blocker code). A "false positive" is any
+ * recorded block whose outcome was later resolved (merged or overridden).
+ */
+export function buildGateFalsePositiveReport(
+  outcomes: GateOutcomeRecord[],
+  repoFullName: string | null = null,
+): GateFalsePositiveReport {
+  const byCode = new Map<string, { blocked: number; falsePositives: number }>();
+  let totalBlocked = 0;
+  let totalFalsePositives = 0;
+
+  for (const outcome of outcomes) {
+    totalBlocked += 1;
+    const falsePositive = outcome.resolution != null;
+    if (falsePositive) totalFalsePositives += 1;
+    for (const code of new Set(outcome.blockerCodes)) {
+      const entry = byCode.get(code) ?? { blocked: 0, falsePositives: 0 };
+      entry.blocked += 1;
+      if (falsePositive) entry.falsePositives += 1;
+      byCode.set(code, entry);
+    }
+  }
+
+  const byGateType: GateFalsePositiveRate[] = [...byCode.entries()]
+    .map(([code, entry]) => ({
+      code,
+      blocked: entry.blocked,
+      falsePositives: entry.falsePositives,
+      falsePositiveRate: rate(entry.falsePositives, entry.blocked),
+    }))
+    .sort((left, right) => left.code.localeCompare(right.code));
+
+  return {
+    repoFullName,
+    totalBlocked,
+    totalFalsePositives,
+    falsePositiveRate: rate(totalFalsePositives, totalBlocked),
+    byGateType,
+  };
+}
+
+function rate(numerator: number, denominator: number): number {
+  return denominator === 0 ? 0 : Math.round((numerator / denominator) * 1000) / 1000;
+}

src/types.ts

@@ -1,4 +1,4 @@
 export type JsonPrimitive = string | number | boolean | null;
 export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue };
 
 export type JobMessage =
@@ -462,6 +462,37 @@
   updatedAt?: string | null | undefined;
 };
 
+// Gate false-positive telemetry (#554). A blocked PR that is later merged or overridden is a false
+// positive — the hard block did not reflect a real defect.
+export type GateOutcomeResolution = "merged" | "overridden";
+
+export type GateOutcomeRecord = {
+  repoFullName: string;
+  prNumber: number;
+  gatePack: string;
+  blockerCodes: string[];
+  blockedAt: string;
+  resolution?: GateOutcomeResolution | null | undefined;
+  resolvedAt?: string | null | undefined;
+};
+
+export type GateFalsePositiveRate = {
+  /** Gate blocker code (the gate "type"), e.g. `missing_linked_issue`, `duplicate_pr_risk`. */
+  code: string;
+  blocked: number;
+  falsePositives: number;
+  /** falsePositives / blocked, in [0, 1], rounded to 3 dp. */
+  falsePositiveRate: number;
+};
+
+export type GateFalsePositiveReport = {
+  repoFullName: string | null;
+  totalBlocked: number;
+  totalFalsePositives: number;
+  falsePositiveRate: number;
+  byGateType: GateFalsePositiveRate[];
+};
+
 export type CommandAuthorizationRole = "maintainer" | "collaborator" | "pr_author" | "confirmed_miner";
 
 export type RepositoryCommandAuthorizationPolicy = {

test/integration/api.test.ts

@@ -1,4 +1,4 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { createSessionForGitHubUser, hashToken } from "../../src/auth/security";
 import {
   upsertBounty,
@@ -33,6 +33,8 @@
   createAgentRun,
   replaceAgentActions,
   upsertAgentRecommendationOutcome,
+  recordGateBlockOutcome,
+  resolveGateOutcome,
 } from "../../src/db/repositories";
 import { createApp } from "../../src/api/routes";
 import { clearPublicRepoStatsCacheForTests } from "../../src/github/public";
@@ -193,6 +195,31 @@
     await expect(unavailable.json()).resolves.toMatchObject({ error: "github_repo_stats_unavailable" });
   });
 
+  it("exposes per-gate-type false-positive telemetry on the internal endpoint (#554)", async () => {
+    const app = createApp();
+    const env = createTestEnv();
+
+    await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 1, gatePack: "gittensor", blockerCodes: ["missing_linked_issue"] });
+    await recordGateBlockOutcome(env, { repoFullName: "acme/widgets", prNumber: 2, gatePack: "gittensor", blockerCodes: ["missing_linked_issue", "duplicate_pr_risk"] });
+    await resolveGateOutcome(env, "acme/widgets", 1, "merged"); // blocked-then-merged → false positive
+
+    const unauthorized = await app.request("/v1/internal/repos/acme/widgets/gate-telemetry", {}, env);
+    expect(unauthorized.status).toBe(401);
+
+    const response = await app.request("/v1/internal/repos/acme/widgets/gate-telemetry", { headers: internalHeaders(env) }, env);
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({
+      repoFullName: "acme/widgets",
+      totalBlocked: 2,
+      totalFalsePositives: 1,
+      falsePositiveRate: 0.5,
+      byGateType: expect.arrayContaining([
+        { code: "missing_linked_issue", blocked: 2, falsePositives: 1, falsePositiveRate: 0.5 },
+        { code: "duplicate_pr_risk", blocked: 1, falsePositives: 0, falsePositiveRate: 0 },
+      ]),
+    });
+  });
+
   it("serves the public README badge only for installed, opted-in repos (#541)", async () => {
     const app = createApp();
     const env = createTestEnv();