diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 182ae975..dd77a6b4 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -21,6 +21,10 @@ import { Prometheus } from "@/lib/prometheus"; import { SpecSchema, type Spec } from "@/lib/spec-schema"; import { renderBenchmarkText } from "@/lib/bench-template"; import { liveResults as liveProviderResults } from "@/lib/provider-filters"; +import { + classifyHealth, + aggregateConfidence, +} from "@/lib/sample-health"; /** Overridable so the worker can run with a different cwd. */ const SPECS_DIR = @@ -266,6 +270,26 @@ export async function specToBenchmark( // surfaces fall back to the coarser bestPerChain path. const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined; + // Per-provider sample-health classification. When the spec declares + // expected_n, every live provider gets `dataConfidence` (healthy / + // low / insufficient) + `sampleHealth` (raw ratio) so the renderer + // can badge undersized rows and so citable APIs can refuse to crown + // a leader drawn from a degraded sample. Bench-wide aggregate is + // the median of the per-provider classifications. + const expectedN = spec.expected_n; + if (expectedN) { + for (const r of live.results) { + const h = classifyHealth(r.sampleSize, expectedN); + if (h) { + r.dataConfidence = h.confidence; + r.sampleHealth = h.ratio; + } + } + } + const agg = expectedN + ? aggregateConfidence(live.results, expectedN) + : undefined; + // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders // against the freshly loaded numbers so editorial text (findings, // seo_intro, faq) never drifts from the displayed data. @@ -276,6 +300,8 @@ export async function specToBenchmark( worstPerChain, providersPerChain, cellRanks, + expectedN, + dataConfidence: agg?.confidence, }); // Persistence is the caller's concern (site: KV snapshot write, // worker: store publish). Only the unfiltered "All" view of a live diff --git a/src/lib/sample-health.test.ts b/src/lib/sample-health.test.ts new file mode 100644 index 00000000..cf372ef6 --- /dev/null +++ b/src/lib/sample-health.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, test } from "bun:test"; +import { + aggregateConfidence, + classifyHealth, + formatHealthPct, + HEALTHY_THRESHOLD, + LOW_THRESHOLD, +} from "./sample-health"; +import type { ProviderResult } from "@/types/benchmark"; + +function row( + slug: string, + sampleSize: number | undefined, + availability: ProviderResult["availability"] = "live", +): ProviderResult { + return { + name: slug, + slug, + ms: { p50: 1, p90: 1, p99: 1, mean: 1 }, + successRate: 100, + sampleSize, + availability, + }; +} + +describe("classifyHealth", () => { + test("returns undefined when expectedN is missing", () => { + expect(classifyHealth(100, undefined)).toBeUndefined(); + expect(classifyHealth(100, 0)).toBeUndefined(); + }); + + test("returns undefined when sampleSize is missing", () => { + expect(classifyHealth(undefined, 1000)).toBeUndefined(); + expect(classifyHealth(NaN, 1000)).toBeUndefined(); + }); + + test("classifies healthy at and above 0.5", () => { + expect(classifyHealth(500, 1000)?.confidence).toBe("healthy"); + expect(classifyHealth(1000, 1000)?.confidence).toBe("healthy"); + expect(classifyHealth(10_000, 1000)?.confidence).toBe("healthy"); + }); + + test("classifies low between 0.1 and 0.5", () => { + expect(classifyHealth(100, 1000)?.confidence).toBe("low"); + expect(classifyHealth(499, 1000)?.confidence).toBe("low"); + }); + + test("classifies insufficient below 0.1", () => { + expect(classifyHealth(0, 1000)?.confidence).toBe("insufficient"); + expect(classifyHealth(99, 1000)?.confidence).toBe("insufficient"); + }); + + test("matches the documented thresholds", () => { + expect(HEALTHY_THRESHOLD).toBe(0.5); + expect(LOW_THRESHOLD).toBe(0.1); + }); + + test("handles low-cadence per-bench expected_n (perp-funding case)", () => { + // perp-funding declares expected_n: 3 (BTC, ETH, SOL series per venue). + expect(classifyHealth(3, 3)?.confidence).toBe("healthy"); + expect(classifyHealth(2, 3)?.confidence).toBe("healthy"); // 0.67 + expect(classifyHealth(1, 3)?.confidence).toBe("low"); // 0.33 + expect(classifyHealth(0, 3)?.confidence).toBe("insufficient"); + }); + + test("handles 4-source oracle-deviation case", () => { + expect(classifyHealth(4, 4)?.confidence).toBe("healthy"); + expect(classifyHealth(3, 4)?.confidence).toBe("healthy"); // 0.75 (XRP/ADA/DOGE) + expect(classifyHealth(1, 4)?.confidence).toBe("low"); // 0.25 + }); +}); + +describe("aggregateConfidence", () => { + test("returns undefined when expectedN is missing", () => { + const results = [row("a", 1000)]; + expect(aggregateConfidence(results, undefined)).toBeUndefined(); + }); + + test("ignores unavailable providers", () => { + const results = [ + row("a", 1000, "unavailable"), + row("b", 1000, "live"), + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy"); + }); + + test("returns undefined when no live provider carries a sample count", () => { + const results = [row("a", undefined), row("b", undefined)]; + expect(aggregateConfidence(results, 1000)).toBeUndefined(); + }); + + test("uses the median ratio so one healthy provider does not mask a bad field", () => { + const results = [ + row("a", 10), // 0.01 insufficient + row("b", 50), // 0.05 insufficient + row("c", 10_000), // healthy + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe( + "insufficient", + ); + }); + + test("a healthy median wins regardless of outliers", () => { + const results = [ + row("a", 0), // insufficient + row("b", 1000), // healthy + row("c", 5000), // healthy + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy"); + }); +}); + +describe("formatHealthPct", () => { + test("formats ratios as integer percent", () => { + expect(formatHealthPct(0.75)).toBe("75%"); + expect(formatHealthPct(0.5)).toBe("50%"); + expect(formatHealthPct(0.09)).toBe("9%"); + }); + + test("clamps very large ratios for display", () => { + expect(formatHealthPct(50)).toBe("999%"); + }); + + test("returns n/a for missing ratios", () => { + expect(formatHealthPct(undefined)).toBe("n/a"); + expect(formatHealthPct(NaN)).toBe("n/a"); + }); +}); diff --git a/src/lib/sample-health.ts b/src/lib/sample-health.ts new file mode 100644 index 00000000..9dc750a0 --- /dev/null +++ b/src/lib/sample-health.ts @@ -0,0 +1,92 @@ +/** + * Sample-health classification. + * + * Bench pages used to rank providers from sample counts spanning 3 to + * tens of thousands without surfacing the confidence level, while the + * methodology page advertised "n >= 1000 per provider". On low cadence + * benches like perp-funding (3 series per venue by design) the methodology + * line read as broken, an E-E-A-T penalty waiting to happen. + * + * A flat threshold would mis-classify legitimately low-cadence benches. + * Per-bench `expected_n` declared in the YAML lets each bench publish + * what "full coverage" looks like, and this module computes + * health = sampleSize / expectedN per provider: + * + * - healthy health >= 0.5 + * - low 0.1 <= health < 0.5 + * - insufficient health < 0.1 + * + * Renderers gate UI accordingly: hide insufficient rows from rankings, + * show low rows with a "Low sample" pill, healthy rows render normally. + * + * Citation surfaces (/api/citable, /api/stat, headline sentences) read + * the bench-level aggregate so a benchmark with mostly-insufficient + * providers does not assert a winner in machine-readable feeds. + */ + +import type { ProviderResult } from "@/types/benchmark"; + +/** Inclusive ratio at and above which a provider is treated as healthy. */ +export const HEALTHY_THRESHOLD = 0.5; +/** Inclusive ratio at and above which a provider is treated as low. */ +export const LOW_THRESHOLD = 0.1; + +export type DataConfidence = "healthy" | "low" | "insufficient"; + +/** Classify a single provider's sample health against the bench-declared + * expected sample count. Returns undefined when the bench did not declare + * expected_n (legacy display behavior, no badge) or when the provider's + * sample count is missing. */ +export function classifyHealth( + sampleSize: number | undefined, + expectedN: number | undefined, +): { confidence: DataConfidence; ratio: number } | undefined { + if (!expectedN || expectedN <= 0) return undefined; + if (sampleSize == null || !Number.isFinite(sampleSize)) return undefined; + const ratio = sampleSize / expectedN; + let confidence: DataConfidence; + if (ratio < LOW_THRESHOLD) confidence = "insufficient"; + else if (ratio < HEALTHY_THRESHOLD) confidence = "low"; + else confidence = "healthy"; + return { confidence, ratio }; +} + +/** + * Aggregate confidence across a provider set, used at the bench level + * (citable JSON, headline sentence, structured data). The aggregate is + * the classification of the median per-provider ratio so a single + * low-cadence outlier does not poison the verdict for an otherwise + * healthy field. Returns undefined when no provider carries a + * computable health (no expectedN, or no provider returned data). + */ +export function aggregateConfidence( + results: ProviderResult[], + expectedN: number | undefined, +): { confidence: DataConfidence; ratio: number } | undefined { + if (!expectedN || expectedN <= 0) return undefined; + const ratios: number[] = []; + for (const r of results) { + if (r.availability === "unavailable") continue; + const h = classifyHealth(r.sampleSize, expectedN); + if (h) ratios.push(h.ratio); + } + if (ratios.length === 0) return undefined; + const sorted = [...ratios].sort((a, b) => a - b); + const mid = sorted.length >> 1; + const median = + sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; + let confidence: DataConfidence; + if (median < LOW_THRESHOLD) confidence = "insufficient"; + else if (median < HEALTHY_THRESHOLD) confidence = "low"; + else confidence = "healthy"; + return { confidence, ratio: median }; +} + +/** Human-readable percentage of expected, clamped to a sensible display + * range for tooltips. Returns "n/a" when health is undefined. */ +export function formatHealthPct(ratio: number | undefined): string { + if (ratio == null || !Number.isFinite(ratio)) return "n/a"; + return `${Math.round(Math.min(ratio, 9.99) * 100)}%`; +} diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index fd5781aa..ab57dac6 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -255,6 +255,23 @@ export const SpecSchema = z findings: z.array(seoText(1, 500)).max(40).default([]), source: z.url(), + /* Per-bench expected sample count over the measurement window. + * Powers the sample-health badge on bench pages: low-cadence + * benches like perp-funding legitimately publish 3 samples per 24h, + * so a flat n threshold would falsely flag them as broken. By + * declaring expected_n in the YAML, the page computes + * health = sampleSize / expected_n per provider and tags rows + * below 50 percent as "low sample" or hides rankings below 10 + * percent as "insufficient". Roughly equals + * cadence_per_minute * window_minutes * routes_per_provider. + * Optional. Benches without expected_n behave as before (no badge). */ + expected_n: z + .number() + .int() + .positive() + .optional() + .describe("Expected sample count per provider over the bench's measurement window. Used to badge undersized samples on the page. Roughly cadence_per_minute * window_minutes * routes_per_provider."), + /* Data source. OpenChainBench is a federation: every contributor * declares the Prometheus their harness publishes to. Schema-time * isPublicHttpsUrl + runtime DNS-resolve guard in the Prom client diff --git a/src/lib/spec.ts b/src/lib/spec.ts index e9369440..9e07430a 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -115,6 +115,13 @@ function overlayEditorial(stored: Benchmark, spec: Spec): Benchmark { // and the bench page filters without waiting on the materialise // worker to rewrite the snapshot. dimensions: spec.dimensions ?? stored.dimensions, + // expected_n is a YAML editorial declaration too: drives the + // sample-health badge logic on the page + the citable APIs. A + // freshly added/edited value must take effect immediately, before + // the worker re-publishes the snapshot, otherwise a bench keeps + // ranking 3-sample providers as healthy through the materialise + // lag. + expectedN: spec.expected_n ?? stored.expectedN, }; // Resolve `{{p50:slug}}`, `{{name:slug}}`, `{{best_name}}` etc. in the // overlaid editorial text. Without this, a YAML edit that ships AHEAD @@ -225,7 +232,11 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // "gram" after the June 2026 rebrand). Without bumping, a v12 entry // would keep serving result.slug = "ton" + result.name = "TON" until // the materialize worker rewrites the snapshot. - ["bench-unfiltered-v13"], + // v14: per-provider dataConfidence + sampleHealth + bench-wide + // expectedN + dataConfidence aggregate (sample-health system). Cached + // v13 entries lack these fields, so the sample-health badge would + // not render on existing benches until the cache aged out. + ["bench-unfiltered-v14"], { revalidate: 300, tags: ["benchmarks"] }, ); @@ -356,7 +367,11 @@ const loadAllBenchmarksCached = unstable_cache( // v17: bumped with bench-unfiltered-v13 (slug reconciliation in // overlayEditorial). Without this, /api/citable and the products // page would keep serving v16 benches with stale "ton" results. - ["all-benchmarks-v17"], + // v18: bumped with bench-unfiltered-v14 (sample-health system). + // Without this, /api/citable + products + sitemap surfaces would + // keep serving v17 benches without the new dataConfidence / + // sampleHealth / expectedN fields. + ["all-benchmarks-v18"], { revalidate: 300, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); diff --git a/src/lib/time-constants.ts b/src/lib/time-constants.ts new file mode 100644 index 00000000..1a3f41bb --- /dev/null +++ b/src/lib/time-constants.ts @@ -0,0 +1,20 @@ +/** + * Shared time-unit constants. + * + * Single source of truth for the milliseconds / seconds conversions that + * leak across files as magic numbers (86400, 60_000, 3_600_000, ...). + * Importing from here makes the intent obvious at the call site and + * keeps grep-ability if a duration unit ever has to change. + * + * Naming convention mirrors the unit-of-measure that the constant + * EXPANDS into (MS_PER_MINUTE === 60_000 milliseconds in one minute). + */ + +export const MS_PER_SECOND = 1_000; +export const MS_PER_MINUTE = 60_000; +export const MS_PER_HOUR = 3_600_000; +export const MS_PER_DAY = 86_400_000; + +export const SECONDS_PER_MINUTE = 60; +export const SECONDS_PER_HOUR = 3_600; +export const SECONDS_PER_DAY = 86_400; diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index e583cfc9..1a8392f1 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -43,6 +43,26 @@ export type ProviderResult = { successRate: number; /** Per-provider sample count over the run window. */ sampleSize?: number; + /** + * Sample-health classification derived from sampleSize / expectedN. + * + * - "healthy" : sampleSize >= 0.5 × expectedN. Render normally. + * - "low" : 0.1 × expectedN <= sampleSize < 0.5 × expectedN. + * Row stays in the ranking with a "Low sample" + * pill and a tooltip explaining the gap. + * - "insufficient": sampleSize < 0.1 × expectedN. Row drops out of + * the sorted ranking; aggregate citations should + * read "insufficient data" rather than assert a + * winner. + * + * Absent on benches whose spec does not declare `expected_n`, in + * which case no badge is rendered (legacy display behavior). */ + dataConfidence?: "healthy" | "low" | "insufficient"; + /** Sample-health ratio, 0..1+. Same source as `dataConfidence` but + * exposed as the raw fraction for downstream consumers (citable / + * stat APIs, dashboards, tooltips). Absent when the spec declares + * no expected_n or when sampleSize itself is missing. */ + sampleHealth?: number; secondary?: { label: string; value: string }; /** Defaults to "live" when the provider returns numbers; the spec * loader sets "unavailable" when prom has no data for the p50 / p90 / @@ -156,6 +176,15 @@ export type Benchmark = { * published-but-awaiting-data bench remains visible. */ editorialStatus: "live" | "draft"; sampleSize: number; + /** Spec-declared expected sample count per provider over the bench's + * window. Drives the per-provider sample-health badge logic. Absent + * when the spec author chose not to declare it (low cadence benches + * whose healthy n cannot be computed deterministically). */ + expectedN?: number; + /** Aggregate sample-health for the bench. Derived from the median of + * the per-provider healths; "insufficient" silences the leader + * assertion at every citable surface. Absent when expectedN is. */ + dataConfidence?: "healthy" | "low" | "insufficient"; abstract: string; metric: string; unit: "ms" | "s" | "sec" | "pct" | "bps" | "bp" | "count" | "slots" | "usd";