JSONbored · RealDiligent · Jul 23, 2026
@@ -0,0 +1,101 @@
+---
+title: Verify this review
+description: Re-run LoopOver's published backtest corpus yourself — download a checksummed snapshot, replay the same scorer, and compare against the published numbers.
+eyebrow: Core concepts
+---
+
+## Why this page exists
+
+LoopOver publishes measured per-rule precision on the [fairness report](/fairness). Numbers on a
+website only build trust if a skeptic can check them without asking anyone's permission. This
+page is the end-to-end walkthrough: export the same corpus snapshot the numbers come from, verify
+its checksum, replay the same scorer over it, and compare what you get against what is published.
+
+Everything below runs read-only against a database export and pure functions from
+`@loopover/engine`. Nothing posts anywhere, nothing needs an API key.
+
+## 1. Export the corpus snapshot
+
+Every rule's fired/override history exports as a versioned, checksummed JSON snapshot
+([backtest &amp; calibration](/docs/backtest-calibration) explains how that history is recorded):
+
+```bash
+npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output corpus.json --remote
+```
+
+On a self-host deployment, point the same CLI at your own Postgres instead:
+
+```bash
+npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output corpus.json --pg "$DATABASE_URL"
+```
+
+The snapshot's `checksum` field is a SHA-256 over the canonicalized cases (keys sorted, so
+property order can never change the hash). The fairness report's *reproducibility freeze point*
+shows the checksum of the corpus behind the latest persisted backtest run — an export of the same
+window reproduces the same checksum, byte for byte.
+
+## 2. Verify the checksum
+
+The manifest is self-verifying: recompute the hash over its own `cases` array and compare it to
+the recorded `checksum`. The canonicalization lives in `scripts/backtest-corpus-export-core.ts`
+(`buildBacktestCorpusManifest`), so the check is one short script:
+
+```bash
+node --experimental-strip-types -e '
+import { readFileSync } from "node:fs";
+import { buildBacktestCorpusManifest } from "./scripts/backtest-corpus-export-core.ts";
+const saved = JSON.parse(readFileSync("corpus.json", "utf8"));
+const recomputed = buildBacktestCorpusManifest(saved.ruleId, saved.cases);
+console.log(recomputed.checksum === saved.checksum ? "checksum OK" : "CHECKSUM MISMATCH");
+'
+```
+
+## 3. Replay the scorer
+
+The published precision comes from the same pure functions any Node script can import:
+`scoreBacktest` replays a classifier over the labeled cases, and `compareBacktestScores` applies
+the Pareto-floor verdict between two scores. Replaying the shipped confidence floor over your
+verified snapshot:
+
+```bash
+node --experimental-strip-types -e '
+import { readFileSync } from "node:fs";
+import { buildConfidenceThresholdClassifier, scoreBacktest } from "@loopover/engine";
+const saved = JSON.parse(readFileSync("corpus.json", "utf8"));
+const report = scoreBacktest(saved.ruleId, saved.cases, buildConfidenceThresholdClassifier(0.5));
+console.log(report);
+'
+```
+
+- **"Reversed" is the positive class** — a prediction of `reversed` says the rule's original
+  firing was wrong, and it is scored against what a human actually decided.
+- **`null` is never `0`.** Precision and recall stay `null` below the decided-sample floor;
+  the fairness report renders that as *insufficient data*, never as a zero.
+
+## 4. Compare against the published numbers
+
+The [fairness report](/fairness) renders each rule's decided-case count and measured precision
+from the public stats endpoint (`/v1/public/stats`, the `rulePrecision` block). The aggregated
+run history is also readable directly:
+
+```bash
+npx tsx scripts/backtest-track-record.ts --db loopover --remote
+```
+
+Your replayed `confirmed / decided` for a rule should match the published precision for the same
+window; the freeze-point checksum ties the published numbers to the exact corpus you just
+verified.
+
+## What this proves — and what it does not
+
+<Callout variant="note">
+  **Proved:** the published scores are real computations over a real, checksummed, replayable
+  corpus — not hand-entered numbers. Anyone can independently reproduce them from the snapshot.
+
+  **Not proved:** that the live gate *ran this exact code* when it made its decisions. Verifying
+  the runtime itself is an attestation problem — a trusted-execution boundary, not a replay
+  boundary — and is tracked as its own explicitly-scoped decision in
+  [#8136](https://github.com/JSONbored/loopover/issues/8136) and
+  [#8137](https://github.com/JSONbored/loopover/issues/8137). This walkthrough is honest about
+  stopping at the reproducibility line rather than implying the stronger guarantee.
+</Callout>
@@ -62,6 +62,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [
   { label: "Upstream drift", to: "/docs/upstream-drift", group: "Docs" },
   { label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" },
   { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" },
+  { label: "Verify this review", to: "/docs/verify-this-review", group: "Docs" },
   { label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" },
   { label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" },
   { label: "API reference", to: "/api", group: "Reference" },

@@ -101,6 +101,7 @@ export const docsNav: DocsGroup[] = [
       { to: "/docs/scoreability", label: "Scoreability" },
       { to: "/docs/upstream-drift", label: "Upstream drift" },
       { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
+      { to: "/docs/verify-this-review", label: "Verify this review" },
     ],
   },
   {

@@ -11,6 +11,24 @@ vi.mock("@/lib/api/request", () => ({
 }));
 vi.mock("@/lib/api/origin", () => ({ getApiOrigin: () => "https://api.example.test" }));
 
+// Mirrors proof-of-power-stats.test.tsx: <Link> needs a real router context; render a plain <a>.
+vi.mock("@tanstack/react-router", () => ({
+  Link: ({
+    to,
+    children,
+    ...props
+  }: {
+    to: string;
+    children: ReactNode;
+    className?: string;
+    "aria-label"?: string;
+  }) => (
+    <a href={to} {...props}>
+      {children}
+    </a>
+  ),
+}));
+
 import { FairnessReportPage } from "./fairness-report-page";
 import type { PublicStats } from "./proof-of-power-stats-model";
 
@@ -51,6 +69,47 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
     apiFetch.mockReset();
   });
 
+  it("renders the measured per-rule precision table with the insufficient-data null state — never 0% (#8231)", async () => {
+    apiFetch.mockResolvedValue({
+      ok: true,
+      data: {
+        ...FIXTURE,
+        rulePrecision: {
+          windowDays: 90,
+          rules: [
+            { ruleId: "linked_issue_scope_mismatch", decided: 42, precision: 0.952 },
+            { ruleId: "slop_gate_score", decided: 3, precision: null },
+          ],
+          reversals: { reopened: 2, reverted: 1, superseded: 0 },
+          latestBacktestRun: { corpusChecksum: "a".repeat(64), at: "2026-07-22T00:00:00.000Z" },
+        },
+      },
+      durationMs: 10,
+    });
+    renderWithClient(<FairnessReportPage />);
+
+    await waitFor(() => expect(screen.getByText("Measured accuracy per rule")).toBeTruthy());
+    expect(screen.getByText("linked_issue_scope_mismatch")).toBeTruthy();
+    expect(screen.getByText("95.2%")).toBeTruthy();
+    // The below-floor rule renders the deliberate null state — the literal words, not a zero.
+    expect(screen.getAllByText("insufficient data").length).toBeGreaterThanOrEqual(2); // the explainer + the table cell
+    expect(screen.queryByText("0%")).toBeNull();
+    // The reproducibility freeze point surfaces the truncated corpus checksum.
+    expect(screen.getByText(/Reproducibility freeze point/)).toBeTruthy();
+    expect(screen.getByText(/aaaaaaaaaaaaaaaa…/)).toBeTruthy();
+    // And the walkthrough link points at the docs page.
+    expect(screen.getByRole("link", { name: /verify this review/i })).toBeTruthy();
+  });
+
+  it("hides the per-rule section entirely when the API response predates rulePrecision (deployment skew) or has no rules (#8231)", async () => {
+    apiFetch.mockResolvedValue({ ok: true, data: FIXTURE, durationMs: 10 });
+    renderWithClient(<FairnessReportPage />);
+    await waitFor(() =>
+      expect(screen.getByText("Is ORB treating contributors fairly?")).toBeTruthy(),
+    );
+    expect(screen.queryByText("Measured accuracy per rule")).toBeNull();
+  });
+
   it("renders a content-shaped loading skeleton", () => {
     apiFetch.mockReturnValue(new Promise(() => {}));
     const { container } = renderWithClient(<FairnessReportPage />);

@@ -1,4 +1,5 @@
 import { useQuery } from "@tanstack/react-query";
+import { Link } from "@tanstack/react-router";
 
 import { getApiOrigin } from "@/lib/api/origin";
 import { apiFetch } from "@/lib/api/request";
@@ -251,6 +252,71 @@ export function FairnessReportPage() {
                 </table>
               </TableScroll>
             </div>
+
+            {data.rulePrecision && data.rulePrecision.rules.length > 0 ? (
+              <div className="mt-10">
+                <h2 className="text-token-lg font-medium">Measured accuracy per rule</h2>
+                <p className="mt-2 text-token-sm text-muted-foreground">
+                  Precision of each automated rule over its human-decided cases in the last{" "}
+                  {data.rulePrecision.windowDays} days. A rule below the decided-sample floor shows{" "}
+                  <span className="font-medium text-foreground">insufficient data</span> — an
+                  unknown is never rendered as 0%. Reproduce these numbers yourself:{" "}
+                  <Link
+                    to="/docs/$slug"
+                    params={{ slug: "verify-this-review" }}
+                    className="underline underline-offset-2"
+                  >
+                    verify this review
+                  </Link>
+                  .
+                </p>
+                <TableScroll className="mt-4" label="Measured precision per rule">
+                  <table className="w-full min-w-[28rem] text-left text-token-sm">
+                    <caption className="sr-only">
+                      Decided cases and measured precision per rule.
+                    </caption>
+                    <thead className="text-token-xs text-muted-foreground">
+                      <tr>
+                        <th scope="col" className="pb-2 pr-4 font-medium">
+                          Rule
+                        </th>
+                        <th scope="col" className="pb-2 pr-4 font-medium">
+                          Decided
+                        </th>
+                        <th scope="col" className="pb-2 font-medium">
+                          Precision
+                        </th>
+                      </tr>
+                    </thead>
+                    <tbody>
+                      {data.rulePrecision.rules.map((row) => (
+                        <tr key={row.ruleId} className="border-t border-hairline">
+                          <td className="py-2 pr-4 font-mono text-token-xs">{row.ruleId}</td>
+                          <td className="py-2 pr-4">{intFmt.format(row.decided)}</td>
+                          <td className="py-2">
+                            {row.precision != null ? (
+                              `${pctFmt.format(row.precision * 100)}%`
+                            ) : (
+                              <span className="text-muted-foreground">insufficient data</span>
+                            )}
+                          </td>
+                        </tr>
+                      ))}
+                    </tbody>
+                  </table>
+                </TableScroll>
+                {data.rulePrecision.latestBacktestRun ? (
+                  <p className="mt-3 text-token-xs text-muted-foreground">
+                    Reproducibility freeze point: corpus checksum{" "}
+                    <span className="font-mono">
+                      {data.rulePrecision.latestBacktestRun.corpusChecksum.slice(0, 16)}…
+                    </span>{" "}
+                    from the latest persisted backtest run (
+                    {new Date(data.rulePrecision.latestBacktestRun.at).toLocaleDateString()}).
+                  </p>
+                ) : null}
+              </div>
+            ) : null}
           </div>
         ) : null}
       </StateBoundary>

@@ -58,6 +58,21 @@ export type PublicStats = {
     merged: number;
     filteredPct: number | null;
   }>;
+
+  /** Measured per-rule precision + the reproducibility freeze point (#8230/#8231). Optional-chained by
+   *  consumers: until the backend carrying it is deployed, an older /v1/public/stats response simply won't
+   *  have the field yet, and every surface must degrade to hiding the section rather than throw. */
+  rulePrecision?: {
+    windowDays: number;
+    rules: Array<{
+      ruleId: string;
+      decided: number;
+      /** confirmed / decided; null below the decided-sample floor -- rendered as "insufficient data", NEVER 0%. */
+      precision: number | null;
+    }>;
+    reversals: { reopened: number; reverted: number; superseded: number };
+    latestBacktestRun: { corpusChecksum: string; at: string } | null;
+  };
 };
 
 /** Relative "updated Ns ago" label from the payload's updatedAt (mirrors MetaStrip's freshness logic). */

@@ -95,6 +95,7 @@ const AUDIENCES: Audience[] = [
       { to: "/docs/upstream-drift", label: "Upstream drift" },
       { to: "/docs/ai-summaries", label: "AI summaries policy" },
       { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
+      { to: "/docs/verify-this-review", label: "Verify this review" },
     ],
   },
   {