From 874393fe6642de06184c637467b1b2d89ecdd77a Mon Sep 17 00:00:00 2001 From: RealDiligent Date: Thu, 23 Jul 2026 22:45:16 +0800 Subject: [PATCH] docs+ui: add 'Verify this review' walkthrough and measured per-rule accuracy on the fairness report Adds the public reproducibility walkthrough doc (export corpus snapshot, verify checksum, replay scorer, compare against published numbers, with an honest attestation-boundary callout) and renders the measured accuracy per rule section on the fairness report from the public stats rulePrecision block: decided cases, precision with the insufficient-data null state (never 0%), reversal counts, and the corpus-checksum freeze point linking to the walkthrough. Closes #8231 --- .../content/docs/verify-this-review.mdx | 101 ++++++++++++++++++ .../src/components/site/command-palette.tsx | 1 + .../src/components/site/docs-nav.tsx | 1 + .../site/fairness-report-page.test.tsx | 59 ++++++++++ .../components/site/fairness-report-page.tsx | 66 ++++++++++++ .../site/proof-of-power-stats-model.ts | 15 +++ apps/loopover-ui/src/routes/docs.index.tsx | 1 + 7 files changed, 244 insertions(+) create mode 100644 apps/loopover-ui/content/docs/verify-this-review.mdx diff --git a/apps/loopover-ui/content/docs/verify-this-review.mdx b/apps/loopover-ui/content/docs/verify-this-review.mdx new file mode 100644 index 0000000000..36a3c6df5f --- /dev/null +++ b/apps/loopover-ui/content/docs/verify-this-review.mdx @@ -0,0 +1,101 @@ +--- +title: Verify this review +description: Re-run LoopOver's published backtest corpus yourself — download a checksummed snapshot, replay the same scorer, and compare against the published numbers. +eyebrow: Core concepts +--- + +## Why this page exists + +LoopOver publishes measured per-rule precision on the [fairness report](/fairness). Numbers on a +website only build trust if a skeptic can check them without asking anyone's permission. This +page is the end-to-end walkthrough: export the same corpus snapshot the numbers come from, verify +its checksum, replay the same scorer over it, and compare what you get against what is published. + +Everything below runs read-only against a database export and pure functions from +`@loopover/engine`. Nothing posts anywhere, nothing needs an API key. + +## 1. Export the corpus snapshot + +Every rule's fired/override history exports as a versioned, checksummed JSON snapshot +([backtest & calibration](/docs/backtest-calibration) explains how that history is recorded): + +```bash +npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output corpus.json --remote +``` + +On a self-host deployment, point the same CLI at your own Postgres instead: + +```bash +npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output corpus.json --pg "$DATABASE_URL" +``` + +The snapshot's `checksum` field is a SHA-256 over the canonicalized cases (keys sorted, so +property order can never change the hash). The fairness report's *reproducibility freeze point* +shows the checksum of the corpus behind the latest persisted backtest run — an export of the same +window reproduces the same checksum, byte for byte. + +## 2. Verify the checksum + +The manifest is self-verifying: recompute the hash over its own `cases` array and compare it to +the recorded `checksum`. The canonicalization lives in `scripts/backtest-corpus-export-core.ts` +(`buildBacktestCorpusManifest`), so the check is one short script: + +```bash +node --experimental-strip-types -e ' +import { readFileSync } from "node:fs"; +import { buildBacktestCorpusManifest } from "./scripts/backtest-corpus-export-core.ts"; +const saved = JSON.parse(readFileSync("corpus.json", "utf8")); +const recomputed = buildBacktestCorpusManifest(saved.ruleId, saved.cases); +console.log(recomputed.checksum === saved.checksum ? "checksum OK" : "CHECKSUM MISMATCH"); +' +``` + +## 3. Replay the scorer + +The published precision comes from the same pure functions any Node script can import: +`scoreBacktest` replays a classifier over the labeled cases, and `compareBacktestScores` applies +the Pareto-floor verdict between two scores. Replaying the shipped confidence floor over your +verified snapshot: + +```bash +node --experimental-strip-types -e ' +import { readFileSync } from "node:fs"; +import { buildConfidenceThresholdClassifier, scoreBacktest } from "@loopover/engine"; +const saved = JSON.parse(readFileSync("corpus.json", "utf8")); +const report = scoreBacktest(saved.ruleId, saved.cases, buildConfidenceThresholdClassifier(0.5)); +console.log(report); +' +``` + +- **"Reversed" is the positive class** — a prediction of `reversed` says the rule's original + firing was wrong, and it is scored against what a human actually decided. +- **`null` is never `0`.** Precision and recall stay `null` below the decided-sample floor; + the fairness report renders that as *insufficient data*, never as a zero. + +## 4. Compare against the published numbers + +The [fairness report](/fairness) renders each rule's decided-case count and measured precision +from the public stats endpoint (`/v1/public/stats`, the `rulePrecision` block). The aggregated +run history is also readable directly: + +```bash +npx tsx scripts/backtest-track-record.ts --db loopover --remote +``` + +Your replayed `confirmed / decided` for a rule should match the published precision for the same +window; the freeze-point checksum ties the published numbers to the exact corpus you just +verified. + +## What this proves — and what it does not + + + **Proved:** the published scores are real computations over a real, checksummed, replayable + corpus — not hand-entered numbers. Anyone can independently reproduce them from the snapshot. + + **Not proved:** that the live gate *ran this exact code* when it made its decisions. Verifying + the runtime itself is an attestation problem — a trusted-execution boundary, not a replay + boundary — and is tracked as its own explicitly-scoped decision in + [#8136](https://github.com/JSONbored/loopover/issues/8136) and + [#8137](https://github.com/JSONbored/loopover/issues/8137). This walkthrough is honest about + stopping at the reproducibility line rather than implying the stronger guarantee. + diff --git a/apps/loopover-ui/src/components/site/command-palette.tsx b/apps/loopover-ui/src/components/site/command-palette.tsx index 527f38a611..ebb7c6f962 100644 --- a/apps/loopover-ui/src/components/site/command-palette.tsx +++ b/apps/loopover-ui/src/components/site/command-palette.tsx @@ -62,6 +62,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [ { label: "Upstream drift", to: "/docs/upstream-drift", group: "Docs" }, { label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" }, { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" }, + { label: "Verify this review", to: "/docs/verify-this-review", group: "Docs" }, { label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" }, { label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" }, { label: "API reference", to: "/api", group: "Reference" }, diff --git a/apps/loopover-ui/src/components/site/docs-nav.tsx b/apps/loopover-ui/src/components/site/docs-nav.tsx index d5a5f415dc..4ffaf87cf6 100644 --- a/apps/loopover-ui/src/components/site/docs-nav.tsx +++ b/apps/loopover-ui/src/components/site/docs-nav.tsx @@ -101,6 +101,7 @@ export const docsNav: DocsGroup[] = [ { to: "/docs/scoreability", label: "Scoreability" }, { to: "/docs/upstream-drift", label: "Upstream drift" }, { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, + { to: "/docs/verify-this-review", label: "Verify this review" }, ], }, { diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx index 12b2bffdec..577ce81f39 100644 --- a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx +++ b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx @@ -11,6 +11,24 @@ vi.mock("@/lib/api/request", () => ({ })); vi.mock("@/lib/api/origin", () => ({ getApiOrigin: () => "https://api.example.test" })); +// Mirrors proof-of-power-stats.test.tsx: needs a real router context; render a plain . +vi.mock("@tanstack/react-router", () => ({ + Link: ({ + to, + children, + ...props + }: { + to: string; + children: ReactNode; + className?: string; + "aria-label"?: string; + }) => ( + + {children} + + ), +})); + import { FairnessReportPage } from "./fairness-report-page"; import type { PublicStats } from "./proof-of-power-stats-model"; @@ -51,6 +69,47 @@ describe("FairnessReportPage (#fairness-analytics)", () => { apiFetch.mockReset(); }); + it("renders the measured per-rule precision table with the insufficient-data null state — never 0% (#8231)", async () => { + apiFetch.mockResolvedValue({ + ok: true, + data: { + ...FIXTURE, + rulePrecision: { + windowDays: 90, + rules: [ + { ruleId: "linked_issue_scope_mismatch", decided: 42, precision: 0.952 }, + { ruleId: "slop_gate_score", decided: 3, precision: null }, + ], + reversals: { reopened: 2, reverted: 1, superseded: 0 }, + latestBacktestRun: { corpusChecksum: "a".repeat(64), at: "2026-07-22T00:00:00.000Z" }, + }, + }, + durationMs: 10, + }); + renderWithClient(); + + await waitFor(() => expect(screen.getByText("Measured accuracy per rule")).toBeTruthy()); + expect(screen.getByText("linked_issue_scope_mismatch")).toBeTruthy(); + expect(screen.getByText("95.2%")).toBeTruthy(); + // The below-floor rule renders the deliberate null state — the literal words, not a zero. + expect(screen.getAllByText("insufficient data").length).toBeGreaterThanOrEqual(2); // the explainer + the table cell + expect(screen.queryByText("0%")).toBeNull(); + // The reproducibility freeze point surfaces the truncated corpus checksum. + expect(screen.getByText(/Reproducibility freeze point/)).toBeTruthy(); + expect(screen.getByText(/aaaaaaaaaaaaaaaa…/)).toBeTruthy(); + // And the walkthrough link points at the docs page. + expect(screen.getByRole("link", { name: /verify this review/i })).toBeTruthy(); + }); + + it("hides the per-rule section entirely when the API response predates rulePrecision (deployment skew) or has no rules (#8231)", async () => { + apiFetch.mockResolvedValue({ ok: true, data: FIXTURE, durationMs: 10 }); + renderWithClient(); + await waitFor(() => + expect(screen.getByText("Is ORB treating contributors fairly?")).toBeTruthy(), + ); + expect(screen.queryByText("Measured accuracy per rule")).toBeNull(); + }); + it("renders a content-shaped loading skeleton", () => { apiFetch.mockReturnValue(new Promise(() => {})); const { container } = renderWithClient(); diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.tsx index f386353520..1cf46e61b2 100644 --- a/apps/loopover-ui/src/components/site/fairness-report-page.tsx +++ b/apps/loopover-ui/src/components/site/fairness-report-page.tsx @@ -1,4 +1,5 @@ import { useQuery } from "@tanstack/react-query"; +import { Link } from "@tanstack/react-router"; import { getApiOrigin } from "@/lib/api/origin"; import { apiFetch } from "@/lib/api/request"; @@ -251,6 +252,71 @@ export function FairnessReportPage() { + + {data.rulePrecision && data.rulePrecision.rules.length > 0 ? ( +
+

Measured accuracy per rule

+

+ Precision of each automated rule over its human-decided cases in the last{" "} + {data.rulePrecision.windowDays} days. A rule below the decided-sample floor shows{" "} + insufficient data — an + unknown is never rendered as 0%. Reproduce these numbers yourself:{" "} + + verify this review + + . +

+ + + + + + + + + + + + {data.rulePrecision.rules.map((row) => ( + + + + + + ))} + +
+ Decided cases and measured precision per rule. +
+ Rule + + Decided + + Precision +
{row.ruleId}{intFmt.format(row.decided)} + {row.precision != null ? ( + `${pctFmt.format(row.precision * 100)}%` + ) : ( + insufficient data + )} +
+ + {data.rulePrecision.latestBacktestRun ? ( +

+ Reproducibility freeze point: corpus checksum{" "} + + {data.rulePrecision.latestBacktestRun.corpusChecksum.slice(0, 16)}… + {" "} + from the latest persisted backtest run ( + {new Date(data.rulePrecision.latestBacktestRun.at).toLocaleDateString()}). +

+ ) : null} +
+ ) : null} ) : null} diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts index 56a60951c0..6b75fa41d7 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts @@ -58,6 +58,21 @@ export type PublicStats = { merged: number; filteredPct: number | null; }>; + + /** Measured per-rule precision + the reproducibility freeze point (#8230/#8231). Optional-chained by + * consumers: until the backend carrying it is deployed, an older /v1/public/stats response simply won't + * have the field yet, and every surface must degrade to hiding the section rather than throw. */ + rulePrecision?: { + windowDays: number; + rules: Array<{ + ruleId: string; + decided: number; + /** confirmed / decided; null below the decided-sample floor -- rendered as "insufficient data", NEVER 0%. */ + precision: number | null; + }>; + reversals: { reopened: number; reverted: number; superseded: number }; + latestBacktestRun: { corpusChecksum: string; at: string } | null; + }; }; /** Relative "updated Ns ago" label from the payload's updatedAt (mirrors MetaStrip's freshness logic). */ diff --git a/apps/loopover-ui/src/routes/docs.index.tsx b/apps/loopover-ui/src/routes/docs.index.tsx index 7faddfce4d..534f45e2b5 100644 --- a/apps/loopover-ui/src/routes/docs.index.tsx +++ b/apps/loopover-ui/src/routes/docs.index.tsx @@ -95,6 +95,7 @@ const AUDIENCES: Audience[] = [ { to: "/docs/upstream-drift", label: "Upstream drift" }, { to: "/docs/ai-summaries", label: "AI summaries policy" }, { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, + { to: "/docs/verify-this-review", label: "Verify this review" }, ], }, {