From 53f761eea03ab78e36e91bf5870ccae8f80f20a6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 11 Jul 2026 05:32:02 -0700 Subject: [PATCH] feat(review): fleet-wide gaming-pattern detector for the self-host orb fleet (#2350) Extends computeFleetAnalytics (src/orb/analytics.ts) with a more targeted detection signal than the existing single-metric outlier check: an instance whose decided-PR volume is unusually high, whose merge-precision is unusually high, AND whose reversal-rate is unusually low -- all three simultaneously -- fits the exact "mass-submitting only trivially-safe PRs to inflate merge-precision" signature the issue describes. A high-precision instance alone isn't suspicious (could be a genuinely careful team); combined with abnormal volume and suspiciously few reversals, it is. Detection only: gamingPatternFlags is a new read-only field surfaced on the existing operator dashboard tile set and the operator-only MCP gittensory_get_fleet_analytics tool summary -- nothing acts on it automatically, nothing here touches the live gate, and instanceId is the same opaque HMAC-derived handle already used throughout this pipeline (orb-collector.ts), never a login or anything more identifying. Scope note (see the module's own doc comment): this flags a self-hosted INSTANCE, never an individual miner -- the fleet pipeline carries no per-actor identity by deliberate, repeatedly-documented design (review_audit has no login column; predicted_gate_calibration_ledger is explicitly never-exported, citing this issue as the reason why). A genuine per-miner detector would require adding a new anonymized per-actor signal to the export pipeline, a separate privacy-sensitive design decision deserving its own focused issue. Also out of scope: "duplicate-claim-election win-rate skew" (isDuplicateClusterWinnerByClaim) is not implemented. Its outcome is never persisted anywhere in this pipeline -- only the losing side of a duplicate cluster produces a finding, with no cluster id and no actor linkage, so there is no winner data to measure a win-rate from. No proxy is implemented; a misleading one would be worse than none. Advances #2350 (does not close it -- the win-rate-skew and per-miner sub-deliverables remain genuinely unbuildable from data that exists today) --- src/mcp/server.ts | 2 +- src/orb/analytics.ts | 70 ++++++++++++- src/services/operator-dashboard.ts | 6 ++ test/unit/operator-dashboard.test.ts | 27 +++++ test/unit/orb-analytics.test.ts | 151 +++++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 2 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index a320465755..958fe94386 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -2859,7 +2859,7 @@ export class GittensoryMcp { const report = await computeFleetAnalytics(this.env, input.windowDays !== undefined ? { windowDays: input.windowDays } : {}); const merge = report.fleet.mergePrecision !== null ? `${Math.round(report.fleet.mergePrecision * 100)}%` : "n/a"; return { - summary: `Fleet calibration over ${report.windowDays}d: ${report.instanceCount} instance(s), median merge precision ${merge}, ${report.outliers.length} outlier(s).`, + summary: `Fleet calibration over ${report.windowDays}d: ${report.instanceCount} instance(s), median merge precision ${merge}, ${report.outliers.length} outlier(s), ${report.gamingPatternFlags.length} gaming-pattern flag(s).`, data: report as unknown as Record, }; } diff --git a/src/orb/analytics.ts b/src/orb/analytics.ts index ecc7c984cc..9264d99602 100644 --- a/src/orb/analytics.ts +++ b/src/orb/analytics.ts @@ -1,9 +1,36 @@ // Gittensory Orb (#1255) — fleet calibration ANALYTICS. Reads the anonymized orb_signals collected from // self-hosted instances and derives gate-accuracy metrics across the fleet. Aggregation is median/percentile // (never mean) so a single instance contributing fabricated data cannot move the fleet numbers. +// +// ANTI-FARMING DETECTION (#2350): gamingPatternFlags below extends the existing outlier check with a more targeted, +// ONE-SIDED signal for the specific "gaming" pattern the issue describes -- an instance mass-submitting only +// trivially-safe PRs to inflate its own merge-precision. mergePrecision alone can't distinguish "gamed" from +// "genuinely excellent" (a careful team also has high precision); combining it with UNUSUALLY HIGH volume and +// UNUSUALLY LOW reversal-rate, all three simultaneously, is the actual farming signature: lots of easy merges, +// nothing risky enough to ever get reverted. Detection only — never an automatic action. +// +// SCOPE (explicit non-goals, read before extending): this flags a self-hosted INSTANCE, never an individual +// miner. The fleet pipeline (orb_signals, review_audit's export) carries NO per-actor identity by deliberate, +// repeatedly-stated design (review_audit has no login column; predicted_gate_calibration_ledger is explicitly +// documented as never-exported, citing THIS issue as the reason) -- a genuine per-miner detector would require +// adding a new anonymized per-actor signal to the export pipeline, which is a privacy-sensitive design +// decision deserving its own focused issue/PR, not a rushed addition here. This module never deanonymizes, +// never auto-bans, and never touches the live gate — instanceId here is the SAME opaque, HMAC-derived handle +// already used everywhere else in this pipeline (see selfhost/orb-collector.ts), nothing more identifying. +// +// OUT OF SCOPE: "duplicate-claim-election win-rate skew" (isDuplicateClusterWinnerByClaim, +// src/signals/duplicate-winner.ts) is NOT implemented here. Its outcome is never persisted anywhere in this +// pipeline — only the LOSING side of a duplicate cluster produces a finding (duplicate_pr_risk), bucketed as +// gate_reasoncode_bucket="duplicate_risk" on export with no cluster id and no actor linkage. There is no +// winner marker to measure a win-rate FROM, and a per-instance duplicate_risk rate would measure something +// different (how often THIS instance's own PRs lose a local collision) than "identities farming wins," so no +// proxy for it is implemented — a misleading proxy would be worse than none. const MIN_DECIDED = 5; // an instance needs at least this many decided PRs to count toward the fleet median const OUTLIER_BAND = 0.25; // |instance precision − fleet median| beyond this flags the instance +const GAMING_VOLUME_MULTIPLIER = 2; // an instance's decided count more than this many times the fleet median +const GAMING_PRECISION_BAND = OUTLIER_BAND; // mergePrecision this far ABOVE the fleet median (one-sided) +const GAMING_REVERSAL_RATIO = 0.5; // reversalRate below this fraction of the fleet median /** Per-instance confusion-matrix cell as stored. */ interface Cell { @@ -29,6 +56,20 @@ export interface InstanceMetrics { reversalRate: number; // share of decided PRs a human reversed } +/** #2350: one self-hosted instance whose combined volume/precision/reversal-rate pattern looks like it is + * gaming the fleet-aggregate accuracy signal (see the module doc comment for the exact signature and its + * scope). Detection only — a human reads this, nothing here takes any action automatically. `instanceId` is + * the same opaque, HMAC-derived handle used throughout this pipeline; nothing more identifying is included. */ +export interface GamingPatternFlag { + instanceId: string; + decided: number; + mergePrecision: number; + reversalRate: number; + fleetMedianDecided: number; + fleetMergePrecision: number; + fleetReversalRate: number; +} + export interface FleetAnalytics { windowDays: number; instanceCount: number; // instances meeting MIN_DECIDED @@ -42,6 +83,7 @@ export interface FleetAnalytics { }; instances: InstanceMetrics[]; outliers: Array<{ instanceId: string; metric: string; value: number; fleetMedian: number }>; + gamingPatternFlags: GamingPatternFlag[]; } function median(xs: number[]): number | null { @@ -127,7 +169,7 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe const reg = await env.DB.prepare(`SELECT instance_id FROM orb_instances WHERE registered = 1`).all<{ instance_id: string }>(); registered = new Set((reg.results ?? []).map((r) => r.instance_id)); } catch { - return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [] }; + return { windowDays, instanceCount: 0, fleet: { mergePrecision: null, closePrecision: null, fpRate: null, reversalRate: null, cycleP50Ms: null, cycleP95Ms: null }, instances: [], outliers: [], gamingPatternFlags: [] }; } // Group cells by instance, fold each. @@ -157,6 +199,31 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe } } + // #2350: gamingPatternFlags. Gated on fleetMergeP !== null (at least one eligible instance made a comparable + // merge verdict) — decided/reversalRate are never null per-instance, so once `eligible` is known non-empty + // (implied by fleetMergeP being resolvable), both medians below are guaranteed non-null too. + const gamingPatternFlags: FleetAnalytics["gamingPatternFlags"] = []; + if (fleetMergeP !== null) { + const fleetMedianDecided = median(eligible.map((i) => i.decided))!; + const fleetReversalRate = median(eligible.map((i) => i.reversalRate))!; + for (const i of eligible) { + const highVolume = i.decided > fleetMedianDecided * GAMING_VOLUME_MULTIPLIER; + const highPrecision = i.mergePrecision !== null && i.mergePrecision - fleetMergeP > GAMING_PRECISION_BAND; + const lowReversal = i.reversalRate < fleetReversalRate * GAMING_REVERSAL_RATIO; + if (highVolume && highPrecision && lowReversal) { + gamingPatternFlags.push({ + instanceId: i.instanceId, + decided: i.decided, + mergePrecision: i.mergePrecision!, + reversalRate: i.reversalRate, + fleetMedianDecided, + fleetMergePrecision: fleetMergeP, + fleetReversalRate, + }); + } + } + } + return { windowDays, instanceCount: eligible.length, @@ -170,5 +237,6 @@ export async function computeFleetAnalytics(env: Env, opts: { windowDays?: numbe }, instances, outliers, + gamingPatternFlags, }; } diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index 8b591bae40..660760c7ff 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -181,6 +181,12 @@ export async function buildOperatorDashboardPayload(env: Env): Promise 0 ? `${fleetMetrics.gamingPatternFlags.map((f) => f.instanceId).join(", ")}` : "no gaming pattern detected", + }, ], noiseReduction: [ { diff --git a/test/unit/operator-dashboard.test.ts b/test/unit/operator-dashboard.test.ts index e13f5dc7a2..e81ceb386e 100644 --- a/test/unit/operator-dashboard.test.ts +++ b/test/unit/operator-dashboard.test.ts @@ -127,10 +127,37 @@ describe("operator dashboard payload", () => { expect.arrayContaining([ expect.objectContaining({ label: "Fleet instances", value: "3", delta: "1 outlier(s)" }), expect.objectContaining({ label: "Fleet merge precision", value: "100%" }), + expect.objectContaining({ label: "Fleet gaming-pattern flags", value: "0", delta: "no gaming pattern detected" }), ]), ); }); + it("surfaces a fleet farming flag (#2350) as a dedicated dashboard tile naming the flagged instance", async () => { + const env = createTestEnv(); + let n = 0; + const seed = async (instance: string, count: number, opts: { reversal?: string } = {}): Promise => { + for (let i = 0; i < count; i++) { + await env.DB + .prepare(`INSERT INTO orb_signals (instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag) VALUES (?, ?, ?, 'merge', 'merged', ?)`) + .bind(instance, `r${n}`, `p${n++}`, opts.reversal ?? "none") + .run(); + } + }; + // Two normal instances: decided 10, precision 0.7, reversalRate 0.3 (7 confirmed + 3 reverted). + for (const id of ["normal1", "normal2"]) { + await seed(id, 7); + await seed(id, 3, { reversal: "reverted" }); + } + // Farmer: decided 30 (> 2x the fleet median volume of 10), precision 1.0 (> 0.7 + 0.25), reversalRate 0. + await seed("farmer", 30); + for (const id of ["normal1", "normal2", "farmer"]) { + await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES (?, 1)`).bind(id).run(); + } + const payload = await buildOperatorDashboardPayload(env); + expect(payload.fleetMetrics.gamingPatternFlags.map((f) => f.instanceId)).toEqual(["farmer"]); + expect(payload.metrics).toEqual(expect.arrayContaining([expect.objectContaining({ label: "Fleet gaming-pattern flags", value: "1", delta: "farmer" })])); + }); + it("picks the newest rollup day for adoption insights", () => { const rollups: ProductUsageDailyRollupRecord[] = [ rollup("2026-05-28"), diff --git a/test/unit/orb-analytics.test.ts b/test/unit/orb-analytics.test.ts index d7a11b1a5c..59b76c71c6 100644 --- a/test/unit/orb-analytics.test.ts +++ b/test/unit/orb-analytics.test.ts @@ -159,3 +159,154 @@ describe("computeFleetAnalytics()", () => { expect(a.fleet.cycleP95Ms).toBe(3000); }); }); + +describe("gamingPatternFlags — anti-farming detection (#2350)", () => { + /** 7 confirmed merges + 3 reverted merges: precision 0.7, reversalRate 0.3. */ + async function normalInstance(env: Env, id: string): Promise { + await signals(env, id, 7, { verdict: "merge", outcome: "merged", reversal: "none" }); + await signals(env, id, 3, { verdict: "merge", outcome: "merged", reversal: "reverted" }); + } + + it("a normal-distribution fleet (similar volume/precision/reversal everywhere) produces no flag", async () => { + const env = createTestEnv(); + await normalInstance(env, "a"); + await normalInstance(env, "b"); + await normalInstance(env, "c"); + await register(env, "a", "b", "c"); + const result = await computeFleetAnalytics(env); + expect(result.instanceCount).toBe(3); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("an inflated-trivial-volume instance (high volume + high precision + low reversal, all three) flags", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); // decided 10, precision 0.7, reversalRate 0.3 + await normalInstance(env, "normal2"); + await normalInstance(env, "normal3"); + // 30 decided (> 2x the fleet median of 10), precision 1.0 (> 0.7 + 0.25), reversalRate 0 (< 0.5 * 0.3). + await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" }); + await register(env, "normal1", "normal2", "normal3", "farmer"); + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toHaveLength(1); + const flag = result.gamingPatternFlags[0]!; + expect(flag.instanceId).toBe("farmer"); + expect(flag.decided).toBe(30); + expect(flag.mergePrecision).toBe(1); + expect(flag.reversalRate).toBe(0); + expect(flag.fleetMedianDecided).toBe(10); + expect(flag.fleetMergePrecision).toBeCloseTo(0.7); + expect(flag.fleetReversalRate).toBeCloseTo(0.3); + // No identity beyond the same opaque instance handle used everywhere else in the pipeline. + expect(Object.keys(flag).sort()).toEqual(["decided", "fleetMedianDecided", "fleetMergePrecision", "fleetReversalRate", "instanceId", "mergePrecision", "reversalRate"]); + }); + + it("high precision WITHOUT elevated volume does not flag (precision alone is not the signature)", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); + await normalInstance(env, "normal2"); + // Same volume (10) as the normals, but perfect precision and zero reversals. + await signals(env, "precise", 10, { verdict: "merge", outcome: "merged", reversal: "none" }); + await register(env, "normal1", "normal2", "precise"); + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toEqual([]); + // It IS still caught by the existing, broader outlier check — this test isolates gamingPatternFlags specifically. + expect(result.outliers.map((o) => o.instanceId)).toContain("precise"); + }); + + it("elevated volume WITHOUT elevated precision does not flag (volume alone is not the signature)", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); + await normalInstance(env, "normal2"); + // 30 decided (high volume) but the SAME precision/reversal profile as everyone else. + await signals(env, "busy", 21, { verdict: "merge", outcome: "merged", reversal: "none" }); + await signals(env, "busy", 9, { verdict: "merge", outcome: "merged", reversal: "reverted" }); + + await register(env, "normal1", "normal2", "busy"); + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("elevated volume + elevated precision but NOT a suspiciously low reversal rate does not flag", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); // decided 10, precision 0.7, reversalRate 0.3 + await normalInstance(env, "normal2"); + // 30 decided (high volume). mergePrecision is 25/25 = 1.0 (elevated) -- ONLY the merge-verdict rows count + // toward it. reversalRate is 5/30 ≈ 0.167, from separate close-verdict reopens -- above the 0.5x-fleet- + // median floor (0.15), i.e. NOT suspiciously low, so this must not read as "farming". + await signals(env, "risky", 25, { verdict: "merge", outcome: "merged", reversal: "none" }); + await signals(env, "risky", 5, { verdict: "close", outcome: "closed", reversal: "reopened" }); + await register(env, "normal1", "normal2", "risky"); + + const result = await computeFleetAnalytics(env); + const risky = result.instances.find((i) => i.instanceId === "risky")!; + expect(risky.mergePrecision).toBe(1); + expect(risky.reversalRate).toBeCloseTo(5 / 30); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("an instance with no merge verdicts at all (null mergePrecision) never flags, even alongside a farmer", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); // decided 10 + await normalInstance(env, "normal2"); // decided 10 + await signals(env, "farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" }); + // Every verdict is "close" — mergePrecision is null, so it cannot be flagged on precision regardless of + // volume. Kept at MIN_DECIDED so it counts toward the fleet without shifting the volume median the farmer + // is measured against. + await signals(env, "close-only", 5, { verdict: "close", outcome: "closed", reversal: "none" }); + await register(env, "normal1", "normal2", "farmer", "close-only"); + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags.map((f) => f.instanceId)).toEqual(["farmer"]); + }); + + it("no flags when no eligible instance has any merge verdict (fleetMergeP unresolvable)", async () => { + const env = createTestEnv(); + await signals(env, "a", 10, { verdict: "close", outcome: "closed" }); + await signals(env, "b", 10, { verdict: "close", outcome: "closed" }); + await register(env, "a", "b"); + + const result = await computeFleetAnalytics(env); + expect(result.fleet.mergePrecision).toBeNull(); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("an unregistered instance never flags, even with an extreme farming-shaped pattern", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); + await normalInstance(env, "normal2"); + await signals(env, "unregistered-farmer", 30, { verdict: "merge", outcome: "merged", reversal: "none" }); + await register(env, "normal1", "normal2"); // deliberately NOT registering the farmer + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toEqual([]); + // Still visible per-instance for the operator, same precedent as outliers. + expect(result.instances.map((i) => i.instanceId)).toContain("unregistered-farmer"); + }); + + it("a below-MIN_DECIDED instance never flags, even if registered with an extreme farming-shaped pattern", async () => { + const env = createTestEnv(); + await normalInstance(env, "normal1"); + await normalInstance(env, "normal2"); + // Only 3 decided (< MIN_DECIDED = 5) — excluded from `eligible` regardless of registration. + await signals(env, "tiny-farmer", 3, { verdict: "merge", outcome: "merged", reversal: "none" }); + await register(env, "normal1", "normal2", "tiny-farmer"); + + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("empty store -> empty gamingPatternFlags (not undefined)", async () => { + const env = createTestEnv(); + const result = await computeFleetAnalytics(env); + expect(result.gamingPatternFlags).toEqual([]); + }); + + it("fail-safe on a DB error -> empty gamingPatternFlags", async () => { + const broken = { DB: { prepare: () => ({ bind: () => ({ all: () => Promise.reject(new Error("boom")) }) }) } } as unknown as Env; + const result = await computeFleetAnalytics(broken); + expect(result.gamingPatternFlags).toEqual([]); + }); +});