feat(sample-health): per-provider data confidence + aggregate (cherry-pick A)

src/lib/materialize/load.ts

@@ -21,6 +21,10 @@ import { Prometheus } from "@/lib/prometheus";
 import { SpecSchema, type Spec } from "@/lib/spec-schema";
 import { renderBenchmarkText } from "@/lib/bench-template";
 import { liveResults as liveProviderResults } from "@/lib/provider-filters";
+import {
+  classifyHealth,
+  aggregateConfidence,
+} from "@/lib/sample-health";
 
 /** Overridable so the worker can run with a different cwd. */
 const SPECS_DIR =
@@ -266,6 +270,26 @@ export async function specToBenchmark(
     // surfaces fall back to the coarser bestPerChain path.
     const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined;
 
+    // Per-provider sample-health classification. When the spec declares
+    // expected_n, every live provider gets `dataConfidence` (healthy /
+    // low / insufficient) + `sampleHealth` (raw ratio) so the renderer
+    // can badge undersized rows and so citable APIs can refuse to crown
+    // a leader drawn from a degraded sample. Bench-wide aggregate is
+    // the median of the per-provider classifications.
+    const expectedN = spec.expected_n;
+    if (expectedN) {
+      for (const r of live.results) {
+        const h = classifyHealth(r.sampleSize, expectedN);
+        if (h) {
+          r.dataConfidence = h.confidence;
+          r.sampleHealth = h.ratio;
+        }
+      }
+    }
+    const agg = expectedN
+      ? aggregateConfidence(live.results, expectedN)
+      : undefined;
+
     // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders
     // against the freshly loaded numbers so editorial text (findings,
     // seo_intro, faq) never drifts from the displayed data.
@@ -276,6 +300,8 @@ export async function specToBenchmark(
       worstPerChain,
       providersPerChain,
       cellRanks,
+      expectedN,
+      dataConfidence: agg?.confidence,
     });
     // Persistence is the caller's concern (site: KV snapshot write,
     // worker: store publish). Only the unfiltered "All" view of a live

src/lib/sample-health.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, test } from "bun:test";
+import {
+  aggregateConfidence,
+  classifyHealth,
+  formatHealthPct,
+  HEALTHY_THRESHOLD,
+  LOW_THRESHOLD,
+} from "./sample-health";
+import type { ProviderResult } from "@/types/benchmark";
+
+function row(
+  slug: string,
+  sampleSize: number | undefined,
+  availability: ProviderResult["availability"] = "live",
+): ProviderResult {
+  return {
+    name: slug,
+    slug,
+    ms: { p50: 1, p90: 1, p99: 1, mean: 1 },
+    successRate: 100,
+    sampleSize,
+    availability,
+  };
+}
+
+describe("classifyHealth", () => {
+  test("returns undefined when expectedN is missing", () => {
+    expect(classifyHealth(100, undefined)).toBeUndefined();
+    expect(classifyHealth(100, 0)).toBeUndefined();
+  });
+
+  test("returns undefined when sampleSize is missing", () => {
+    expect(classifyHealth(undefined, 1000)).toBeUndefined();
+    expect(classifyHealth(NaN, 1000)).toBeUndefined();
+  });
+
+  test("classifies healthy at and above 0.5", () => {
+    expect(classifyHealth(500, 1000)?.confidence).toBe("healthy");
+    expect(classifyHealth(1000, 1000)?.confidence).toBe("healthy");
+    expect(classifyHealth(10_000, 1000)?.confidence).toBe("healthy");
+  });
+
+  test("classifies low between 0.1 and 0.5", () => {
+    expect(classifyHealth(100, 1000)?.confidence).toBe("low");
+    expect(classifyHealth(499, 1000)?.confidence).toBe("low");
+  });
+
+  test("classifies insufficient below 0.1", () => {
+    expect(classifyHealth(0, 1000)?.confidence).toBe("insufficient");
+    expect(classifyHealth(99, 1000)?.confidence).toBe("insufficient");
+  });
+
+  test("matches the documented thresholds", () => {
+    expect(HEALTHY_THRESHOLD).toBe(0.5);
+    expect(LOW_THRESHOLD).toBe(0.1);
+  });
+
+  test("handles low-cadence per-bench expected_n (perp-funding case)", () => {
+    // perp-funding declares expected_n: 3 (BTC, ETH, SOL series per venue).
+    expect(classifyHealth(3, 3)?.confidence).toBe("healthy");
+    expect(classifyHealth(2, 3)?.confidence).toBe("healthy"); // 0.67
+    expect(classifyHealth(1, 3)?.confidence).toBe("low"); // 0.33
+    expect(classifyHealth(0, 3)?.confidence).toBe("insufficient");
+  });
+
+  test("handles 4-source oracle-deviation case", () => {
+    expect(classifyHealth(4, 4)?.confidence).toBe("healthy");
+    expect(classifyHealth(3, 4)?.confidence).toBe("healthy"); // 0.75 (XRP/ADA/DOGE)
+    expect(classifyHealth(1, 4)?.confidence).toBe("low"); // 0.25
+  });
+});
+
+describe("aggregateConfidence", () => {
+  test("returns undefined when expectedN is missing", () => {
+    const results = [row("a", 1000)];
+    expect(aggregateConfidence(results, undefined)).toBeUndefined();
+  });
+
+  test("ignores unavailable providers", () => {
+    const results = [
+      row("a", 1000, "unavailable"),
+      row("b", 1000, "live"),
+    ];
+    expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy");
+  });
+
+  test("returns undefined when no live provider carries a sample count", () => {
+    const results = [row("a", undefined), row("b", undefined)];
+    expect(aggregateConfidence(results, 1000)).toBeUndefined();
+  });
+
+  test("uses the median ratio so one healthy provider does not mask a bad field", () => {
+    const results = [
+      row("a", 10), // 0.01 insufficient
+      row("b", 50), // 0.05 insufficient
+      row("c", 10_000), // healthy
+    ];
+    expect(aggregateConfidence(results, 1000)?.confidence).toBe(
+      "insufficient",
+    );
+  });
+
+  test("a healthy median wins regardless of outliers", () => {
+    const results = [
+      row("a", 0), // insufficient
+      row("b", 1000), // healthy
+      row("c", 5000), // healthy
+    ];
+    expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy");
+  });
+});
+
+describe("formatHealthPct", () => {
+  test("formats ratios as integer percent", () => {
+    expect(formatHealthPct(0.75)).toBe("75%");
+    expect(formatHealthPct(0.5)).toBe("50%");
+    expect(formatHealthPct(0.09)).toBe("9%");
+  });
+
+  test("clamps very large ratios for display", () => {
+    expect(formatHealthPct(50)).toBe("999%");
+  });
+
+  test("returns n/a for missing ratios", () => {
+    expect(formatHealthPct(undefined)).toBe("n/a");
+    expect(formatHealthPct(NaN)).toBe("n/a");
+  });
+});

src/lib/sample-health.ts

@@ -0,0 +1,92 @@
+/**
+ * Sample-health classification.
+ *
+ * Bench pages used to rank providers from sample counts spanning 3 to
+ * tens of thousands without surfacing the confidence level, while the
+ * methodology page advertised "n >= 1000 per provider". On low cadence
+ * benches like perp-funding (3 series per venue by design) the methodology
+ * line read as broken, an E-E-A-T penalty waiting to happen.
+ *
+ * A flat threshold would mis-classify legitimately low-cadence benches.
+ * Per-bench `expected_n` declared in the YAML lets each bench publish
+ * what "full coverage" looks like, and this module computes
+ * health = sampleSize / expectedN per provider:
+ *
+ *   - healthy      health >= 0.5
+ *   - low          0.1 <= health < 0.5
+ *   - insufficient health < 0.1
+ *
+ * Renderers gate UI accordingly: hide insufficient rows from rankings,
+ * show low rows with a "Low sample" pill, healthy rows render normally.
+ *
+ * Citation surfaces (/api/citable, /api/stat, headline sentences) read
+ * the bench-level aggregate so a benchmark with mostly-insufficient
+ * providers does not assert a winner in machine-readable feeds.
+ */
+
+import type { ProviderResult } from "@/types/benchmark";
+
+/** Inclusive ratio at and above which a provider is treated as healthy. */
+export const HEALTHY_THRESHOLD = 0.5;
+/** Inclusive ratio at and above which a provider is treated as low. */
+export const LOW_THRESHOLD = 0.1;
+
+export type DataConfidence = "healthy" | "low" | "insufficient";
+
+/** Classify a single provider's sample health against the bench-declared
+ *  expected sample count. Returns undefined when the bench did not declare
+ *  expected_n (legacy display behavior, no badge) or when the provider's
+ *  sample count is missing. */
+export function classifyHealth(
+  sampleSize: number | undefined,
+  expectedN: number | undefined,
+): { confidence: DataConfidence; ratio: number } | undefined {
+  if (!expectedN || expectedN <= 0) return undefined;
+  if (sampleSize == null || !Number.isFinite(sampleSize)) return undefined;
+  const ratio = sampleSize / expectedN;
+  let confidence: DataConfidence;
+  if (ratio < LOW_THRESHOLD) confidence = "insufficient";
+  else if (ratio < HEALTHY_THRESHOLD) confidence = "low";
+  else confidence = "healthy";
+  return { confidence, ratio };
+}
+
+/**
+ * Aggregate confidence across a provider set, used at the bench level
+ * (citable JSON, headline sentence, structured data). The aggregate is
+ * the classification of the median per-provider ratio so a single
+ * low-cadence outlier does not poison the verdict for an otherwise
+ * healthy field. Returns undefined when no provider carries a
+ * computable health (no expectedN, or no provider returned data).
+ */
+export function aggregateConfidence(
+  results: ProviderResult[],
+  expectedN: number | undefined,
+): { confidence: DataConfidence; ratio: number } | undefined {
+  if (!expectedN || expectedN <= 0) return undefined;
+  const ratios: number[] = [];
+  for (const r of results) {
+    if (r.availability === "unavailable") continue;
+    const h = classifyHealth(r.sampleSize, expectedN);
+    if (h) ratios.push(h.ratio);
+  }
+  if (ratios.length === 0) return undefined;
+  const sorted = [...ratios].sort((a, b) => a - b);
+  const mid = sorted.length >> 1;
+  const median =
+    sorted.length % 2 === 0
+      ? (sorted[mid - 1] + sorted[mid]) / 2
+      : sorted[mid];
+  let confidence: DataConfidence;
+  if (median < LOW_THRESHOLD) confidence = "insufficient";
+  else if (median < HEALTHY_THRESHOLD) confidence = "low";
+  else confidence = "healthy";
+  return { confidence, ratio: median };
+}
+
+/** Human-readable percentage of expected, clamped to a sensible display
+ *  range for tooltips. Returns "n/a" when health is undefined. */
+export function formatHealthPct(ratio: number | undefined): string {
+  if (ratio == null || !Number.isFinite(ratio)) return "n/a";
+  return `${Math.round(Math.min(ratio, 9.99) * 100)}%`;
+}

src/lib/spec-schema.ts

@@ -255,6 +255,23 @@ export const SpecSchema = z
     findings: z.array(seoText(1, 500)).max(40).default([]),
     source: z.url(),
 
+    /* Per-bench expected sample count over the measurement window.
+     * Powers the sample-health badge on bench pages: low-cadence
+     * benches like perp-funding legitimately publish 3 samples per 24h,
+     * so a flat n threshold would falsely flag them as broken. By
+     * declaring expected_n in the YAML, the page computes
+     * health = sampleSize / expected_n per provider and tags rows
+     * below 50 percent as "low sample" or hides rankings below 10
+     * percent as "insufficient". Roughly equals
+     * cadence_per_minute * window_minutes * routes_per_provider.
+     * Optional. Benches without expected_n behave as before (no badge). */
+    expected_n: z
+      .number()
+      .int()
+      .positive()
+      .optional()
+      .describe("Expected sample count per provider over the bench's measurement window. Used to badge undersized samples on the page. Roughly cadence_per_minute * window_minutes * routes_per_provider."),
+
     /* Data source. OpenChainBench is a federation: every contributor
      * declares the Prometheus their harness publishes to. Schema-time
      * isPublicHttpsUrl + runtime DNS-resolve guard in the Prom client

src/lib/spec.ts

@@ -115,6 +115,13 @@ function overlayEditorial(stored: Benchmark, spec: Spec): Benchmark {
     // and the bench page filters without waiting on the materialise
     // worker to rewrite the snapshot.
     dimensions: spec.dimensions ?? stored.dimensions,
+    // expected_n is a YAML editorial declaration too: drives the
+    // sample-health badge logic on the page + the citable APIs. A
+    // freshly added/edited value must take effect immediately, before
+    // the worker re-publishes the snapshot, otherwise a bench keeps
+    // ranking 3-sample providers as healthy through the materialise
+    // lag.
+    expectedN: spec.expected_n ?? stored.expectedN,
   };
   // Resolve `{{p50:slug}}`, `{{name:slug}}`, `{{best_name}}` etc. in the
   // overlaid editorial text. Without this, a YAML edit that ships AHEAD
@@ -225,7 +232,11 @@ const loadBenchmarkUnfilteredCached = unstable_cache(
   // "gram" after the June 2026 rebrand). Without bumping, a v12 entry
   // would keep serving result.slug = "ton" + result.name = "TON" until
   // the materialize worker rewrites the snapshot.
-  ["bench-unfiltered-v13"],
+  // v14: per-provider dataConfidence + sampleHealth + bench-wide
+  // expectedN + dataConfidence aggregate (sample-health system). Cached
+  // v13 entries lack these fields, so the sample-health badge would
+  // not render on existing benches until the cache aged out.
+  ["bench-unfiltered-v14"],
   { revalidate: 300, tags: ["benchmarks"] },
 );
 
@@ -356,7 +367,11 @@ const loadAllBenchmarksCached = unstable_cache(
   // v17: bumped with bench-unfiltered-v13 (slug reconciliation in
   // overlayEditorial). Without this, /api/citable and the products
   // page would keep serving v16 benches with stale "ton" results.
-  ["all-benchmarks-v17"],
+  // v18: bumped with bench-unfiltered-v14 (sample-health system).
+  // Without this, /api/citable + products + sitemap surfaces would
+  // keep serving v17 benches without the new dataConfidence /
+  // sampleHealth / expectedN fields.
+  ["all-benchmarks-v18"],
   { revalidate: 300, tags: ["benchmarks"] },
 );
 export const loadAllBenchmarks = cache(loadAllBenchmarksCached);

src/lib/time-constants.ts

@@ -0,0 +1,20 @@
+/**
+ * Shared time-unit constants.
+ *
+ * Single source of truth for the milliseconds / seconds conversions that
+ * leak across files as magic numbers (86400, 60_000, 3_600_000, ...).
+ * Importing from here makes the intent obvious at the call site and
+ * keeps grep-ability if a duration unit ever has to change.
+ *
+ * Naming convention mirrors the unit-of-measure that the constant
+ * EXPANDS into (MS_PER_MINUTE === 60_000 milliseconds in one minute).
+ */
+
+export const MS_PER_SECOND = 1_000;
+export const MS_PER_MINUTE = 60_000;
+export const MS_PER_HOUR = 3_600_000;
+export const MS_PER_DAY = 86_400_000;
+
+export const SECONDS_PER_MINUTE = 60;
+export const SECONDS_PER_HOUR = 3_600;
+export const SECONDS_PER_DAY = 86_400;

src/types/benchmark.ts

@@ -43,6 +43,26 @@ export type ProviderResult = {
   successRate: number;
   /** Per-provider sample count over the run window. */
   sampleSize?: number;
+  /**
+   * Sample-health classification derived from sampleSize / expectedN.
+   *
+   *  - "healthy"     : sampleSize >= 0.5 × expectedN. Render normally.
+   *  - "low"         : 0.1 × expectedN <= sampleSize < 0.5 × expectedN.
+   *                    Row stays in the ranking with a "Low sample"
+   *                    pill and a tooltip explaining the gap.
+   *  - "insufficient": sampleSize < 0.1 × expectedN. Row drops out of
+   *                    the sorted ranking; aggregate citations should
+   *                    read "insufficient data" rather than assert a
+   *                    winner.
+   *
+   * Absent on benches whose spec does not declare `expected_n`, in
+   * which case no badge is rendered (legacy display behavior). */
+  dataConfidence?: "healthy" | "low" | "insufficient";
+  /** Sample-health ratio, 0..1+. Same source as `dataConfidence` but
+   *  exposed as the raw fraction for downstream consumers (citable /
+   *  stat APIs, dashboards, tooltips). Absent when the spec declares
+   *  no expected_n or when sampleSize itself is missing. */
+  sampleHealth?: number;
   secondary?: { label: string; value: string };
   /** Defaults to "live" when the provider returns numbers; the spec
    *  loader sets "unavailable" when prom has no data for the p50 / p90 /
@@ -156,6 +176,15 @@ export type Benchmark = {
    *  published-but-awaiting-data bench remains visible. */
   editorialStatus: "live" | "draft";
   sampleSize: number;
+  /** Spec-declared expected sample count per provider over the bench's
+   *  window. Drives the per-provider sample-health badge logic. Absent
+   *  when the spec author chose not to declare it (low cadence benches
+   *  whose healthy n cannot be computed deterministically). */
+  expectedN?: number;
+  /** Aggregate sample-health for the bench. Derived from the median of
+   *  the per-provider healths; "insufficient" silences the leader
+   *  assertion at every citable surface. Absent when expectedN is. */
+  dataConfidence?: "healthy" | "low" | "insufficient";
   abstract: string;
   metric: string;
   unit: "ms" | "s" | "sec" | "pct" | "bps" | "bp" | "count" | "slots" | "usd";