From 57d762be79687c5b49be117d4e013c992e118da9 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Mon, 22 Jun 2026 16:59:15 +0200 Subject: [PATCH 1/2] feat(hf): card metadata + schema completeness v2 (audit follow-up) --- .../hf_publisher/dataset_template/README.md | 567 ++++++++++++++++-- .../schemas/chain_leaders.schema.json | 27 + .../schemas/headlines.schema.json | 39 +- .../schemas/providers.schema.json | 31 +- .../schemas/timeseries.schema.json | 3 +- scripts/hf_publisher/publish.py | 235 ++++++-- scripts/hf_publisher/test_publish.py | 131 +++- src/app/api/stat/[slug]/route.ts | 6 + 8 files changed, 905 insertions(+), 134 deletions(-) create mode 100644 scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json diff --git a/scripts/hf_publisher/dataset_template/README.md b/scripts/hf_publisher/dataset_template/README.md index 3b75792a..c49465f7 100644 --- a/scripts/hf_publisher/dataset_template/README.md +++ b/scripts/hf_publisher/dataset_template/README.md @@ -2,10 +2,22 @@ license: cc-by-4.0 language: - en -pretty_name: OpenChainBench Benchmarks +language_creators: + - machine-generated +annotations_creators: + - machine-generated +multilinguality: + - monolingual +pretty_name: OpenChainBench Crypto Infrastructure Benchmarks +viewer: true +source_datasets: + - original task_categories: - - tabular-classification + - time-series-forecasting + - tabular-regression - other +task_ids: + - univariate-time-series-forecasting tags: - crypto - blockchain @@ -17,8 +29,20 @@ tags: - infrastructure - latency - finance + - defi + - solana + - ethereum + - hyperliquid + - mev + - observability + - sla + - mlcroissant + - tabular + - timeseries + - monitoring size_categories: - - 1K`. + +### Languages + +The dataset is monolingual: English (`en`). All text fields (title, +category, metric, unit) are English. Provider names are project +trademarks and preserve their original casing. + +## Dataset Structure + +### Data Instances + +Each config is a flat Parquet table partitioned by `snapshot_date`. +A row looks like (headlines): + +```json +{ + "snapshot_date": "2026-06-22", + "captured_at": "2026-06-22T14:37:00+00:00", + "slug": "bridge-quote-latency", + "title": "Bridge Quote Latency", + "category": "Bridges", + "metric": "Quote Latency", + "unit": "ms", + "status": "live", + "value": 412.0, + "higher_is_better": false, + "leader_name": "LI.FI", + "leader_slug": "lifi", + "leader_value": 412.0, + "bench_sample_size": 12480.0, + "as_of": "2026-06-22T14:30:00.000Z", + "citation_url": "https://openchainbench.com/benchmarks/bridge-quote-latency", + "stat_api_url": "https://openchainbench.com/api/stat/bridge-quote-latency", + "source_url": "https://github.com/ChainBench/OpenChainBench/blob/main/benchmarks/bridge-quote-latency.yml", + "license": "CC-BY-4.0", + "schema_version": 2 +} +``` + +### Data Fields + +Each config has its own schema. JSON Schema files live alongside this +README under `schemas/`. Tables below are the authoritative source for +column names and nullability. + +#### Data Fields, headlines + +One row per (slug, snapshot_date). The "who leads" feed. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key, ISO date (UTC) of the capture | `2026-06-22` | +| captured_at | string | no | ISO 8601 timestamp of the capture | `2026-06-22T14:37:00+00:00` | +| slug | string | no | Benchmark slug, stable URL identifier | `bridge-quote-latency` | +| title | string | no | Human-readable benchmark title | `Bridge Quote Latency` | +| category | string | no | One of `RPCs`, `Bridges`, `Blockchains`, `Aggregators`, `Trading`, `Wallets`, `NFT APIs` | `Bridges` | +| metric | string | no | What is measured | `Quote Latency` | +| unit | string | no | One of `ms`, `s`, `sec`, `pct`, `bps`, `bp`, `count`, `slots`, `usd` | `ms` | +| status | string | no | One of `live`, `draft`, `insufficient` | `live` | +| value | float64 | yes | Headline value of the leader. Null when `status != live` | `412.0` | +| higher_is_better | bool | yes | Direction of the metric, sourced from `/api/stat`. Null when the per-slug fetch failed | `false` | +| leader_name | string | yes | Display name of the leading provider | `LI.FI` | +| leader_slug | string | yes | URL-safe slug of the leading provider | `lifi` | +| leader_value | float64 | yes | Leader's value in `unit` | `412.0` | +| bench_sample_size | float64 | yes | Aggregate sample count over the bench's run window | `12480.0` | +| as_of | string | yes | Source-side timestamp of the underlying measurement | `2026-06-22T14:30:00.000Z` | +| citation_url | string | no | Canonical citation URL for the benchmark | `https://openchainbench.com/benchmarks/bridge-quote-latency` | +| stat_api_url | string | no | Per-bench live JSON endpoint | `https://openchainbench.com/api/stat/bridge-quote-latency` | +| source_url | string | yes | URL of the bench YAML spec in this repo | `https://github.com/ChainBench/OpenChainBench/blob/main/benchmarks/bridge-quote-latency.yml` | +| license | string | no | Always `CC-BY-4.0` for the data | `CC-BY-4.0` | +| schema_version | int64 | no | Additive schema epoch, bumped on new columns | `2` | + +#### Data Fields, providers + +One row per (bench, provider, snapshot_date). Per-provider rankings. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `bridge-quote-latency` | +| provider_name | string | no | Display name of the provider | `LI.FI` | +| provider_slug | string | no | URL-safe provider slug, stable across snapshots | `lifi` | +| provider_type | string | yes | Architectural category (e.g. `aggregator`, `node-rpc`, `oracle`) | `aggregator` | +| provider_layer | string | yes | Network layer when declared (`L1`, `L2`, etc.) | `L1` | +| provider_tag | string | yes | Free-form tag from the bench YAML | `premium` | +| p50 | float64 | yes | 50th percentile in the bench's `unit` | `412.0` | +| p90 | float64 | yes | 90th percentile | `780.0` | +| p99 | float64 | yes | 99th percentile | `1230.0` | +| mean | float64 | yes | Arithmetic mean | `465.3` | +| success_rate | float64 | yes | Fraction (0..1) or percent (0..100) per bench convention | `0.997` | +| provider_sample_size | float64 | yes | Per-provider sample count over the run window | `2080.0` | +| is_leader | bool | no | True for the provider whose slug matches `headlines.leader_slug` | `true` | +| schema_version | int64 | no | Schema epoch | `2` | + +#### Data Fields, timeseries + +One row per (bench, provider, window, point_index, snapshot_date). The +24h, 7d, and 30d trajectories sourced from `/api/series`. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `bridge-quote-latency` | +| provider_slug | string | yes | Provider this point belongs to. Null only on legacy 24h fallback rows that predate per-provider series | `lifi` | +| point_index | int64 | no | Zero-based index inside the window | `42` | +| value | float64 | no | Value of the metric at this point, in the bench's `unit` | `423.7` | +| window | string | no | One of `24h`, `7d`, `30d` | `24h` | +| schema_version | int64 | no | Schema epoch | `2` | + +#### Data Fields, chain_leaders + +One row per (bench, chain, snapshot_date). Per-chain leader and worst +provider for benches whose spec declares a chain dimension. Currently +empty: see "Considerations for Using the Data" for the open task. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `eth-rpc-head-lag` | +| chain | string | no | Chain slug from the bench spec | `ethereum` | +| leader_name | string | yes | Best provider on this chain | `Mobula` | +| leader_slug | string | yes | URL-safe leader slug | `mobula` | +| leader_value | float64 | yes | Leader's value on this chain | `87.2` | +| worst_name | string | yes | Worst provider on this chain | `LegacyRPC` | +| worst_slug | string | yes | URL-safe worst slug | `legacyrpc` | +| worst_value | float64 | yes | Worst provider's value | `1421.3` | +| schema_version | int64 | no | Schema epoch | `2` | + +### Data Splits + +Every config exposes a single `train` split. There is no held-out +evaluation split because the dataset is observational: it records +measurements as they happen and downstream users define their own +train / test cuts (typically by `snapshot_date`). + +## Dataset Creation + +### Curation Rationale + +The OpenChainBench site renders human-readable leaderboards but its +underlying JSON feeds are designed to be agent-friendly: every value +is paired with a methodology link, a license, and a sample size. This +dataset freezes those feeds daily so: + +- LLM agents and journalists can cite a deterministic snapshot. +- ML researchers can train models without depending on a live API + whose numbers move every minute. +- Operators can compare today's leader against arbitrary historical + baselines without rebuilding the harness. + +### Source Data + +#### Initial Data Collection and Normalization + +Raw measurements are collected by per-bench harnesses (open-sourced in +the [OpenChainBench GitHub repo](https://github.com/ChainBench/OpenChainBench) +or, for a few benches, in a private mobula-api repo where they exist +behind paid API keys). Harnesses publish Prometheus metrics that the +OCB Next.js app aggregates into a `Benchmark` object per bench. + +The dataset publisher reads from: + +- `https://openchainbench.com/api/citable` for the headline feed. +- `https://openchainbench.com/api/stat/` for per-bench detail + (provider rankings, sparkline, `higherIsBetter`). +- `https://openchainbench.com/api/series/?range=` for the + 24h / 7d / 30d per-provider trajectories. + +Each per-bench page documents its full methodology. The +`citation_url` column of `headlines` is the stable link to that page. -| Config | Grain | What's in it | -|---|---|---| -| `headlines` | 1 row per (bench, day) | Title, metric, leader, headline value, license, source URL | -| `providers` | 1 row per (bench, provider, day) | p50/p90/p99, mean, success rate, sample size, is_leader flag | -| `timeseries` | 1 row per (bench, point, day) | 24h sparkline values (~72 points per bench) | +#### Who are the source language producers? -All tables are partitioned by `snapshot_date=YYYY-MM-DD`. Each row also -carries a `captured_at` timestamp (UTC, ISO 8601) and a `schema_version` -integer. +All text fields (titles, methodology copy, category labels) are +authored by OpenChainBench contributors in the YAML benchmark specs +under `benchmarks/` in the GitHub repo. The data values themselves are +machine-generated by the measurement harnesses. + +### Annotations + +The dataset has no human-applied annotation layer. Provider rankings +and leader flags are derived programmatically from the percentile +measurements according to each bench's `higher_is_better` direction. + +#### Annotation process + +The OCB Next.js layer computes the leader as the provider with the +best `p50` according to the bench's direction. `bestPerChain` and +`worstPerChain` (when populated) are computed with the same rule +scoped to chain-restricted samples. The `is_leader` boolean in the +`providers` table is a derived projection of `headlines.leader_slug`. + +#### Who are the annotators? + +There are no human annotators. Categorical fields like `category`, +`metric`, `unit`, `provider_type`, and `provider_layer` are authored +by the bench YAML maintainers and reviewed via the same PR process as +the harness code. + +### Personal and Sensitive Information + +The dataset contains no personal or sensitive information. Provider +identifiers refer to operational entities (companies, networks, public +APIs) and are publicly listed on the OpenChainBench site. + +## Considerations for Using the Data + +### Social Impact of Dataset + +Public, reproducible measurements of crypto infrastructure raise the +bar for operator transparency. Downstream consumers should not, however, +treat any single snapshot as definitive: providers' production +characteristics change with traffic, deployments, and incident +recovery. + +### Discussion of Biases + +- **Vantage bias**: latency benchmarks are scraped from a small set of + Prometheus harnesses located in specific cloud regions. The exact + vantage points and methodology are documented per bench at + `citation_url`. +- **Sample asymmetry**: providers that rate-limit our probes hard end + up with smaller `provider_sample_size` than providers that allow + generous quotas. This biases percentile estimates upward (fewer + samples surface tail latency less reliably). The `is_leader` flag is + derived purely from the p50 figure and may therefore reflect + measurement-side asymmetry, not just provider performance. +- **Aggregator coverage**: providers that wrap multiple upstream APIs + (aggregators, with `provider_type = "aggregator"`) compete on a + different surface than single-vendor providers and are not strictly + apples-to-apples comparable. The `provider_type` column is meant to + let consumers filter or stratify by this distinction. + +### Other Known Limitations + +- **Schema stability promise (additive only)**: new columns may be + added without warning. Consumer queries should select named columns + rather than `SELECT *`. Existing columns are never renamed or + removed within a `schema_version`. If a breaking change is + unavoidable, a parallel v3 / v4 folder ships alongside the v2 + partitions so old consumers keep working. +- **`chain_leaders` is currently empty**: the `bestPerChain` / + `worstPerChain` data exists inside the OCB aggregator but is not + exposed by `/api/citable` or `/api/stat` yet. The table is shipped + with its canonical schema so downstream pipelines can stabilize + against a real (zero-row) parquet today and start receiving rows + as soon as the API surfaces the field. Tracked in the publisher's + source code as a `TODO`. +- **24h legacy fallback**: when `/api/series` returns no payload for a + bench's 24h window, the publisher falls back to the aggregate + `sparkline` from `/api/stat`. Those fallback rows carry the leader's + `provider_slug` rather than a per-provider series; downstream users + who care about per-provider trajectories should filter on `window in + ('7d', '30d')` or join `provider_slug` against the `providers` + config. +- **Quorum guard**: if the source feed reports fewer than half its + benches as `live` on capture day, the publisher refuses to upload a + new partition. The previous good snapshot stays as truth that day. + +## Additional Information + +### Dataset Curators + +OpenChainBench Contributors. The publishing pipeline is open source +under the Apache 2.0 license at +[github.com/ChainBench/OpenChainBench](https://github.com/ChainBench/OpenChainBench) +(see `scripts/hf_publisher/`). + +### Licensing Information + +Data is released under +[Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/). +You may use it freely with attribution to OpenChainBench. + +The publishing scripts and benchmark YAML specs are released under +Apache 2.0. + +### Citation Information + +Suggested attribution string: + +> OpenChainBench. (2026). OpenChainBench Crypto Infrastructure Benchmarks +> [Data set]. Hugging Face. +> https://huggingface.co/datasets/OpenChainBench/benchmarks + +BibTeX: + +```bibtex +@dataset{openchainbench_2026, + author = {{OpenChainBench Contributors}}, + title = {OpenChainBench Benchmarks}, + year = {2026}, + publisher = {Hugging Face}, + url = {https://huggingface.co/datasets/OpenChainBench/benchmarks}, + note = {Daily snapshots; DOI minted post-Zenodo sync} +} +``` + +A machine-readable `CITATION.cff` is also published at the root of +this dataset; GitHub, HF, and Zenodo all parse it. + +### Contributions + +Bug reports, schema requests, and new benchmark proposals go through +GitHub Issues at +[github.com/ChainBench/OpenChainBench/issues](https://github.com/ChainBench/OpenChainBench/issues). +Benchmarks are contributed as YAML files plus a Prometheus-emitting +harness; the contributor guide is in +[CONTRIBUTING.md](https://github.com/ChainBench/OpenChainBench/blob/main/CONTRIBUTING.md). ## Quick start @@ -91,52 +587,23 @@ print(latest) ```sql SELECT slug, leader_name, value, unit FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' -WHERE snapshot_date = (SELECT max(snapshot_date) FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet'); +WHERE snapshot_date = ( + SELECT max(snapshot_date) + FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' +); ``` More examples in `examples/`. -## Methodology - -Each benchmark documents its full methodology on the OCB site (per-bench -URL is in the `citation_url` column of `headlines`). Reading list: - -- Per-bench page (open methodology): `https://openchainbench.com/benchmarks/` -- Site-wide methodology: `https://openchainbench.com/methodology` -- YAML spec source (Apache 2.0): `https://github.com/ChainBench/OpenChainBench/tree/main/benchmarks` - -Harness code is open source where applicable (column `source_url`). - -## Schema stability promise - -- New columns may be added without warning. Consumer queries should - select named columns rather than `SELECT *`. -- Existing columns will never be renamed or removed. If a breaking - change ever proves unavoidable, a parallel v2 folder ships alongside - the v1 partitions so old consumers keep working. -- `schema_version` integer in each row marks the additive schema epoch. - ## Update cadence Snapshots run daily at 00:00 UTC via a GitHub Action. If a run aborts on a quorum check (the source feed has fewer than half its benches -live), no new partition is added that day - the previous good snapshot +live), no new partition is added that day. The previous good snapshot stays as truth. -## License & citation - -Data is released under -[Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/). -Use it freely with attribution to OpenChainBench. - -For academic citation see `CITATION.cff` at the root of this dataset -(GitHub / HF / Zenodo all parse it). Suggested attribution string: - -> OpenChainBench. (2026). OpenChainBench Benchmarks [Data set]. -> Hugging Face. https://huggingface.co/datasets/OpenChainBench/benchmarks - ## Provenance - Source code (publisher + benchmark YAML specs): https://github.com/ChainBench/OpenChainBench -- Live measurement APIs: https://openchainbench.com/api/citable, /api/stat, /api/llm-context, /api/mcp +- Live measurement APIs: https://openchainbench.com/api/citable, /api/stat, /api/series, /api/llm-context, /api/mcp - Issues / questions: https://github.com/ChainBench/OpenChainBench/issues diff --git a/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json b/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json new file mode 100644 index 00000000..25552b16 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/chain_leaders.schema.json", + "title": "OCB chain_leaders row", + "description": "One row per (bench, chain, snapshot_date). Per-chain leader and worst provider for benches whose spec declares a chain dimension. Empty until /api/citable or /api/stat exposes bestPerChain / worstPerChain.", + "type": "object", + "required": [ + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "schema_version" + ], + "properties": { + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "bench_slug": { "type": "string" }, + "chain": { "type": "string" }, + "leader_name": { "type": ["string", "null"] }, + "leader_slug": { "type": ["string", "null"] }, + "leader_value": { "type": ["number", "null"] }, + "worst_name": { "type": ["string", "null"] }, + "worst_slug": { "type": ["string", "null"] }, + "worst_value": { "type": ["number", "null"] }, + "schema_version": { "type": "integer", "minimum": 1 } + } +} diff --git a/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json b/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json index 6042ebb6..592cbf28 100644 --- a/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json +++ b/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json @@ -16,24 +16,25 @@ "schema_version" ], "properties": { - "snapshot_date": { "type": "string", "format": "date" }, - "captured_at": { "type": "string", "format": "date-time" }, - "slug": { "type": "string" }, - "title": { "type": "string" }, - "category": { "type": "string" }, - "metric": { "type": "string" }, - "unit": { "type": "string" }, - "status": { "type": "string", "enum": ["live", "draft", "insufficient"] }, - "value": { "type": ["number", "null"] }, - "leader_name": { "type": ["string", "null"] }, - "leader_slug": { "type": ["string", "null"] }, - "leader_value": { "type": ["number", "null"] }, - "sample_size": { "type": ["number", "null"] }, - "as_of": { "type": ["string", "null"], "format": "date-time" }, - "citation_url": { "type": "string", "format": "uri" }, - "stat_api_url": { "type": "string", "format": "uri" }, - "source_url": { "type": ["string", "null"], "format": "uri" }, - "license": { "type": "string", "const": "CC-BY-4.0" }, - "schema_version": { "type": "integer", "minimum": 1 } + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "slug": { "type": "string" }, + "title": { "type": "string" }, + "category": { "type": "string" }, + "metric": { "type": "string" }, + "unit": { "type": "string" }, + "status": { "type": "string", "enum": ["live", "draft", "insufficient"] }, + "value": { "type": ["number", "null"] }, + "higher_is_better": { "type": ["boolean", "null"] }, + "leader_name": { "type": ["string", "null"] }, + "leader_slug": { "type": ["string", "null"] }, + "leader_value": { "type": ["number", "null"] }, + "bench_sample_size": { "type": ["number", "null"] }, + "as_of": { "type": ["string", "null"], "format": "date-time" }, + "citation_url": { "type": "string", "format": "uri" }, + "stat_api_url": { "type": "string", "format": "uri" }, + "source_url": { "type": ["string", "null"], "format": "uri" }, + "license": { "type": "string", "const": "CC-BY-4.0" }, + "schema_version": { "type": "integer", "minimum": 1 } } } diff --git a/scripts/hf_publisher/dataset_template/schemas/providers.schema.json b/scripts/hf_publisher/dataset_template/schemas/providers.schema.json index 55e59959..ee07032c 100644 --- a/scripts/hf_publisher/dataset_template/schemas/providers.schema.json +++ b/scripts/hf_publisher/dataset_template/schemas/providers.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/providers.schema.json", "title": "OCB providers row", - "description": "One row per (bench, provider, snapshot_date). Detailed ranking with percentiles.", + "description": "One row per (bench, provider, snapshot_date). Detailed ranking with percentiles plus provider type / layer / tag classification.", "type": "object", "required": [ "snapshot_date", @@ -12,18 +12,21 @@ "schema_version" ], "properties": { - "snapshot_date": { "type": "string", "format": "date" }, - "captured_at": { "type": "string", "format": "date-time" }, - "bench_slug": { "type": "string" }, - "provider_name": { "type": "string" }, - "provider_slug": { "type": "string" }, - "p50": { "type": ["number", "null"] }, - "p90": { "type": ["number", "null"] }, - "p99": { "type": ["number", "null"] }, - "mean": { "type": ["number", "null"] }, - "success_rate": { "type": ["number", "null"] }, - "sample_size": { "type": ["number", "null"] }, - "is_leader": { "type": "boolean" }, - "schema_version": { "type": "integer", "minimum": 1 } + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "bench_slug": { "type": "string" }, + "provider_name": { "type": "string" }, + "provider_slug": { "type": "string" }, + "provider_type": { "type": ["string", "null"] }, + "provider_layer": { "type": ["string", "null"] }, + "provider_tag": { "type": ["string", "null"] }, + "p50": { "type": ["number", "null"] }, + "p90": { "type": ["number", "null"] }, + "p99": { "type": ["number", "null"] }, + "mean": { "type": ["number", "null"] }, + "success_rate": { "type": ["number", "null"] }, + "provider_sample_size": { "type": ["number", "null"] }, + "is_leader": { "type": "boolean" }, + "schema_version": { "type": "integer", "minimum": 1 } } } diff --git a/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json b/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json index 0f2d3044..90538d02 100644 --- a/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json +++ b/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/timeseries.schema.json", "title": "OCB timeseries row", - "description": "One row per (bench, point_index, snapshot_date). 24h sparkline series.", + "description": "One row per (bench, provider, window, point_index, snapshot_date). 24h / 7d / 30d trajectories sourced from /api/series.", "type": "object", "required": [ "snapshot_date", @@ -17,6 +17,7 @@ "snapshot_date": { "type": "string", "format": "date" }, "captured_at": { "type": "string", "format": "date-time" }, "bench_slug": { "type": "string" }, + "provider_slug": { "type": ["string", "null"] }, "point_index": { "type": "integer", "minimum": 0 }, "value": { "type": "number" }, "window": { "type": "string", "enum": ["24h", "7d", "30d"] }, diff --git a/scripts/hf_publisher/publish.py b/scripts/hf_publisher/publish.py index 9c088ab8..f1800149 100644 --- a/scripts/hf_publisher/publish.py +++ b/scripts/hf_publisher/publish.py @@ -3,18 +3,22 @@ `OpenChainBench/benchmarks`. Reads the live citable JSON API plus per-bench detail, projects it into -three Hive-partitioned Parquet tables (headlines, providers, timeseries) -keyed by snapshot_date, and pushes the new partitions to the HF dataset -repo. - -Why three tables and not one wide table: - headlines - 1 row per (slug, date). Light. The "who leads" feed used - by LLM agents and journalists. Cheap to scan. - providers - 1 row per (slug, provider, date). Detailed per-provider - ranking with p50/p90/p99. Used by devs comparing options. - timeseries - 1 row per (slug, point_index, date) holding the 24h - sparkline. Useful for analytical workloads. Separated so - consumers can ignore it if they only want headlines. +four Hive-partitioned Parquet tables (headlines, providers, timeseries, +chain_leaders) keyed by snapshot_date, and pushes the new partitions to +the HF dataset repo. + +Why multiple tables and not one wide table: + headlines 1 row per (slug, date). Light. The "who leads" feed used + by LLM agents and journalists. Cheap to scan. + providers 1 row per (slug, provider, date). Detailed per-provider + ranking with p50/p90/p99 plus type/layer/tag classification. + timeseries 1 row per (slug, point_index, window, date) holding the + 24h / 7d / 30d trajectories. Separated so consumers can + ignore it if they only want headlines. + chain_leaders 1 row per (slug, chain, date) holding per-chain leader + and worst provider. Empty for benches without a chain + dimension. Currently a placeholder pending /api/citable + or /api/stat exposing bestPerChain. Quorum guard: refuses to publish if /api/citable returns fewer than half its declared count as live. The previous good snapshot stays as truth @@ -22,7 +26,7 @@ Schema versioning: each table embeds a `schema_version` int column. Bump it when adding columns. Never remove columns. Never rename. The -HF dataset is a long-lived public artifact - downstream consumers will +HF dataset is a long-lived public artifact, downstream consumers will write queries that assume column names are stable. Idempotency: same snapshot_date overwrites itself. Re-running the cron @@ -50,13 +54,24 @@ logger = logging.getLogger("hf_publisher") # Bump together with any additive schema change in the row builders below. -# Never decrement. Never re-use a version for a breaking change - if a +# Never decrement. Never re-use a version for a breaking change. If a # breaking change is unavoidable, ship a parallel `headlines_v2/` folder # while keeping the v1 partitions readable for old consumers. -SCHEMA_VERSION = 1 +# +# v1 (2026-06-22): initial release. 3 tables (headlines, providers, +# timeseries). `sample_size` ambiguously named. +# v2 (2026-06-22): adds `higher_is_better` to headlines, renames +# `sample_size` to `bench_sample_size` (headlines) and +# `provider_sample_size` (providers), adds +# `provider_type`, `provider_layer`, `provider_tag` to +# providers, extends timeseries with 7d / 30d windows +# and a `provider_slug` column, and adds the +# `chain_leaders` table (empty until bestPerChain is +# exposed via the public API). +SCHEMA_VERSION = 2 # Minimum count of live benches in /api/citable to allow publishing. The -# bench registry sits around 26; a snapshot with <50% live is considered +# bench registry sits around 26, a snapshot with <50% live is considered # degraded and refused. Floor of 8 avoids tripping during early-stage # dev where the registry is intentionally small. QUORUM_MIN_LIVE = 8 @@ -64,7 +79,13 @@ DEFAULT_API_BASE = "https://openchainbench.com" DEFAULT_REPO_ID = "OpenChainBench/benchmarks" -USER_AGENT = "ocb-hf-publisher/1.0 (+https://openchainbench.com)" +USER_AGENT = "ocb-hf-publisher/2.0 (+https://openchainbench.com)" + +# Time-series windows fetched per bench. `/api/series/?range=` +# returns one series per provider in the leaderboard at that range. We +# fold each provider's series into the parquet so consumers can compute +# trajectories without re-hitting the live API. +TIMESERIES_WINDOWS = ("24h", "7d", "30d") class PublisherError(Exception): @@ -110,25 +131,35 @@ def validate_quorum(citable: dict[str, Any]) -> None: def build_headlines( citable: dict[str, Any], snap: Snapshot, + higher_is_better_by_slug: dict[str, bool] | None = None, ) -> pd.DataFrame: + """Build the headlines table. The `higher_is_better_by_slug` map is + sourced from the per-slug /api/stat fetches done by `run()`. /api/citable + does not surface this field today, so we backfill from the stat + payloads. Benches whose stat fetch failed get a null entry, which is + intentional since we cannot interpret their leader without it. + """ + higher_is_better_by_slug = higher_is_better_by_slug or {} rows: list[dict[str, Any]] = [] for b in citable.get("benchmarks", []): leader = b.get("leader") or {} + slug = b.get("slug") rows.append( { "snapshot_date": snap.date, "captured_at": snap.captured_at, - "slug": b.get("slug"), + "slug": slug, "title": b.get("title"), "category": b.get("category"), "metric": b.get("metric"), "unit": b.get("unit"), "status": b.get("status"), "value": _f(b.get("value")), + "higher_is_better": higher_is_better_by_slug.get(slug), "leader_name": leader.get("name"), "leader_slug": leader.get("slug"), "leader_value": _f(leader.get("value")), - "sample_size": _f(b.get("sampleSize")), + "bench_sample_size": _f(b.get("sampleSize")), "as_of": b.get("asOf"), "citation_url": b.get("url"), "stat_api_url": b.get("api"), @@ -157,12 +188,15 @@ def build_providers( "bench_slug": slug, "provider_name": r.get("name"), "provider_slug": r.get("slug"), + "provider_type": r.get("type"), + "provider_layer": r.get("layer"), + "provider_tag": r.get("tag"), "p50": _f(ms.get("p50")), "p90": _f(ms.get("p90")), "p99": _f(ms.get("p99")), "mean": _f(ms.get("mean")), "success_rate": _f(r.get("successRate")), - "sample_size": _f(r.get("sampleSize")), + "provider_sample_size": _f(r.get("sampleSize")), "is_leader": r.get("slug") == leader_slug, "schema_version": SCHEMA_VERSION, } @@ -172,29 +206,111 @@ def build_providers( def build_timeseries( stats: Iterable[dict[str, Any]], + series_by_slug: dict[str, dict[str, dict[str, Any]]], snap: Snapshot, ) -> pd.DataFrame: + """Build the timeseries table from /api/series payloads. + + `series_by_slug[slug][window]` is the JSON payload from + /api/series/?range=. The 24h window also falls back + to the sparkline embedded in /api/stat when /api/series 404s, so + timeseries stays populated for benches whose series endpoint has no + data yet. The 7d / 30d windows have no such fallback. The bench + silently emits zero rows for them if /api/series said no_data. + """ rows: list[dict[str, Any]] = [] - for stat in stats: - slug = stat.get("slug") - spark = stat.get("sparkline") or [] - for idx, value in enumerate(spark): - if value is None: - continue - rows.append( - { - "snapshot_date": snap.date, - "captured_at": snap.captured_at, - "bench_slug": slug, - "point_index": idx, - "value": _f(value), - "window": "24h", - "schema_version": SCHEMA_VERSION, - } - ) + stats_by_slug = {s.get("slug"): s for s in stats if s.get("slug")} + + def _payload_has_data(payload: dict[str, Any] | None) -> bool: + if not payload: + return False + for prov in payload.get("providers") or []: + values = prov.get("values") or [] + if any(v is not None for v in values): + return True + return False + + for slug, by_window in series_by_slug.items(): + for window, payload in by_window.items(): + providers = payload.get("providers") or [] + for prov in providers: + provider_slug = prov.get("slug") + values = prov.get("values") or [] + for idx, value in enumerate(values): + if value is None: + continue + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": slug, + "provider_slug": provider_slug, + "point_index": idx, + "value": _f(value), + "window": window, + "schema_version": SCHEMA_VERSION, + } + ) + # Fallback for 24h: use the sparkline from /api/stat when + # /api/series returned nothing or only empty providers for the + # 24h range. Avoids losing historical 24h coverage on benches + # whose /api/series endpoint hasn't been wired up yet. + if not _payload_has_data(by_window.get("24h")): + stat = stats_by_slug.get(slug) or {} + spark = stat.get("sparkline") or [] + leader_slug = (stat.get("leader") or {}).get("slug") + for idx, value in enumerate(spark): + if value is None: + continue + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": slug, + "provider_slug": leader_slug, + "point_index": idx, + "value": _f(value), + "window": "24h", + "schema_version": SCHEMA_VERSION, + } + ) return pd.DataFrame(rows) +def build_chain_leaders( + citable: dict[str, Any], + snap: Snapshot, +) -> pd.DataFrame: + """Build the chain_leaders table. + + TODO: requires /api/citable or /api/stat extension to expose + bestPerChain / worstPerChain. Until then this table is intentionally + empty so the schema, partitions, and downstream queries can stabilize + against a real (zero-row) parquet. The schema is documented in + `schemas/chain_leaders.schema.json`. + """ + rows: list[dict[str, Any]] = [] + # No-op iteration kept for the day the API exposes the field. + for _b in citable.get("benchmarks", []): + pass + # Empty DataFrame with the canonical column order so the parquet + # carries its schema even when no rows are produced. + columns = [ + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "leader_name", + "leader_slug", + "leader_value", + "worst_name", + "worst_slug", + "worst_value", + "schema_version", + ] + return pd.DataFrame(rows, columns=columns) + + def _f(v: Any) -> float | None: if v is None: return None @@ -290,6 +406,20 @@ def stage_static_assets(out_root: Path, template_root: Path, snap: Snapshot) -> shutil.copy2(src, dst) +def fetch_series(api_base: str, slug: str) -> dict[str, dict[str, Any]]: + """Fetch /api/series/?range= for every supported window. + Returns a dict {window: payload}. Windows that 404 (no data for + range) are skipped silently.""" + out: dict[str, dict[str, Any]] = {} + for window in TIMESERIES_WINDOWS: + url = f"{api_base}/api/series/{slug}?range={window}" + try: + out[window] = fetch_json(url) + except PublisherError as e: + logger.info("skip series %s @ %s: %s", slug, window, e) + return out + + def run( api_base: str, repo_id: str, @@ -314,23 +444,37 @@ def run( if b.get("status") == "live" and b.get("slug") ] stats: list[dict[str, Any]] = [] + series_by_slug: dict[str, dict[str, dict[str, Any]]] = {} for slug in live_slugs: try: stats.append(fetch_json(f"{api_base}/api/stat/{slug}")) except PublisherError as e: # One bad per-slug fetch shouldn't abort the run. Log and skip. logger.warning("skip per-slug fetch for %s: %s", slug, e) - - headlines = build_headlines(citable, snap) + continue + series_by_slug[slug] = fetch_series(api_base, slug) + + # /api/citable does not surface higherIsBetter today, /api/stat does. + # We backfill from the per-slug payloads so headlines carries the + # field. Slugs whose stat fetch failed get a null entry. + higher_is_better_by_slug = { + s.get("slug"): bool(s.get("higherIsBetter")) + for s in stats + if s.get("slug") and "higherIsBetter" in s + } + + headlines = build_headlines(citable, snap, higher_is_better_by_slug) providers = build_providers(stats, snap) - timeseries = build_timeseries(stats, snap) + timeseries = build_timeseries(stats, series_by_slug, snap) + chain_leaders = build_chain_leaders(citable, snap) if headlines.empty: - raise PublisherError("empty headlines table - refusing to publish") + raise PublisherError("empty headlines table, refusing to publish") write_partition(headlines, out_root, "headlines", snap) write_partition(providers, out_root, "providers", snap) write_partition(timeseries, out_root, "timeseries", snap) + write_partition(chain_leaders, out_root, "chain_leaders", snap) stage_static_assets(out_root, template_root, snap) if dry_run: @@ -340,13 +484,18 @@ def run( if not token: raise PublisherError("HF_TOKEN missing in non-dry-run mode") - commit_message = f"snapshot {snap.date} (rows: h={len(headlines)} p={len(providers)} ts={len(timeseries)})" + commit_message = ( + f"snapshot {snap.date} " + f"(rows: h={len(headlines)} p={len(providers)} " + f"ts={len(timeseries)} cl={len(chain_leaders)})" + ) oid = push_to_hf(out_root, repo_id, token, commit_message) logger.info("pushed to HF: %s commit=%s", repo_id, oid) post_slack( slack_webhook, f":white_check_mark: OCB HF snapshot {snap.date} published " - f"(h={len(headlines)}, p={len(providers)}, ts={len(timeseries)}) " + f"(h={len(headlines)}, p={len(providers)}, " + f"ts={len(timeseries)}, cl={len(chain_leaders)}) " f"https://huggingface.co/datasets/{repo_id}/tree/main", ) diff --git a/scripts/hf_publisher/test_publish.py b/scripts/hf_publisher/test_publish.py index 911c0b59..20b1f4bf 100644 --- a/scripts/hf_publisher/test_publish.py +++ b/scripts/hf_publisher/test_publish.py @@ -16,6 +16,7 @@ PublisherError, SCHEMA_VERSION, Snapshot, + build_chain_leaders, build_headlines, build_providers, build_timeseries, @@ -61,11 +62,15 @@ def _citable_fixture(live_count: int = 20, total: int = 26) -> dict: def _stat_fixture(slug: str = "bench-0", n_providers: int = 3, sparkline_len: int = 72) -> dict: return { "slug": slug, + "higherIsBetter": False, "leader": {"name": "Mobula", "slug": "mobula"}, "rankings": [ { "name": f"Provider {i}", "slug": f"provider-{i}", + "type": "aggregator" if i == 0 else "node-rpc", + "layer": "L1" if i % 2 == 0 else "L2", + "tag": "premium" if i == 0 else None, "ms": { "p50": 100.0 + i, "p90": 200.0 + i, @@ -81,6 +86,31 @@ def _stat_fixture(slug: str = "bench-0", n_providers: int = 3, sparkline_len: in } +def _series_fixture( + slug: str = "bench-0", + windows: tuple[str, ...] = ("24h", "7d", "30d"), + n_providers: int = 2, + points_per_window: dict[str, int] | None = None, +) -> dict[str, dict]: + points_per_window = points_per_window or {"24h": 72, "7d": 84, "30d": 60} + out: dict[str, dict] = {} + for w in windows: + n = points_per_window.get(w, 72) + out[w] = { + "slug": slug, + "range": w, + "providers": [ + { + "slug": f"provider-{i}", + "name": f"Provider {i}", + "values": [float(j + i) for j in range(n)], + } + for i in range(n_providers) + ], + } + return out + + class QuorumTests(unittest.TestCase): def test_passes_at_full_live(self): validate_quorum(_citable_fixture(live_count=26, total=26)) @@ -114,10 +144,11 @@ def test_columns_stable(self): "unit", "status", "value", + "higher_is_better", "leader_name", "leader_slug", "leader_value", - "sample_size", + "bench_sample_size", "as_of", "citation_url", "stat_api_url", @@ -138,6 +169,25 @@ def test_insufficient_rows_have_null_value(self): self.assertTrue(live["value"].notna().all()) self.assertTrue(insufficient["value"].isna().all()) + def test_higher_is_better_backfilled(self): + df = build_headlines( + _citable_fixture(live_count=2, total=2), + _snap(), + higher_is_better_by_slug={"bench-0": True, "bench-1": False}, + ) + by_slug = {r["slug"]: r["higher_is_better"] for _, r in df.iterrows()} + self.assertEqual(by_slug["bench-0"], True) + self.assertEqual(by_slug["bench-1"], False) + + def test_higher_is_better_null_when_missing(self): + df = build_headlines(_citable_fixture(live_count=1, total=1), _snap()) + self.assertIsNone(df.iloc[0]["higher_is_better"]) + + def test_bench_sample_size_renamed(self): + df = build_headlines(_citable_fixture(), _snap()) + self.assertIn("bench_sample_size", df.columns) + self.assertNotIn("sample_size", df.columns) + class ProvidersTests(unittest.TestCase): def test_one_row_per_provider(self): @@ -156,18 +206,79 @@ def test_leader_flag_matches(self): df = build_providers([stat], _snap()) self.assertEqual(df[df["is_leader"]]["provider_slug"].tolist(), ["provider-0"]) + def test_provider_classification_columns(self): + df = build_providers([_stat_fixture(n_providers=3)], _snap()) + for col in ("provider_type", "provider_layer", "provider_tag"): + self.assertIn(col, df.columns) + # First provider has type=aggregator, tag=premium per fixture. + row0 = df[df["provider_slug"] == "provider-0"].iloc[0] + self.assertEqual(row0["provider_type"], "aggregator") + self.assertEqual(row0["provider_tag"], "premium") + + def test_provider_sample_size_renamed(self): + df = build_providers([_stat_fixture(n_providers=2)], _snap()) + self.assertIn("provider_sample_size", df.columns) + self.assertNotIn("sample_size", df.columns) + class TimeseriesTests(unittest.TestCase): - def test_one_row_per_point(self): - df = build_timeseries([_stat_fixture(sparkline_len=72)], _snap()) - self.assertEqual(len(df), 72) + def test_one_row_per_point_per_provider_per_window(self): + series = {"bench-0": _series_fixture(n_providers=2)} + df = build_timeseries([_stat_fixture()], series, _snap()) + # 24h: 72 * 2, 7d: 84 * 2, 30d: 60 * 2 = 432 + self.assertEqual(len(df), (72 + 84 + 60) * 2) + + def test_windows_emitted(self): + series = {"bench-0": _series_fixture(n_providers=1)} + df = build_timeseries([_stat_fixture()], series, _snap()) + self.assertEqual(set(df["window"].unique()), {"24h", "7d", "30d"}) def test_skips_nulls(self): - stat = _stat_fixture(sparkline_len=5) - stat["sparkline"] = [1.0, None, 3.0, None, 5.0] - df = build_timeseries([stat], _snap()) + series = { + "bench-0": { + "24h": { + "providers": [ + {"slug": "p", "values": [1.0, None, 3.0, None, 5.0]} + ] + } + } + } + df = build_timeseries([_stat_fixture()], series, _snap()) self.assertEqual(len(df), 3) + def test_fallback_to_sparkline_when_no_series(self): + # When /api/series returned no payloads, fall back to /api/stat sparkline + # for the 24h window (legacy behaviour). + df = build_timeseries([_stat_fixture(sparkline_len=72)], {"bench-0": {}}, _snap()) + only_24h = df[df["window"] == "24h"] + self.assertEqual(len(only_24h), 72) + + def test_provider_slug_column_present(self): + series = {"bench-0": _series_fixture(n_providers=1)} + df = build_timeseries([_stat_fixture()], series, _snap()) + self.assertIn("provider_slug", df.columns) + + +class ChainLeadersTests(unittest.TestCase): + def test_empty_with_canonical_columns(self): + df = build_chain_leaders(_citable_fixture(), _snap()) + # Empty for now: spec API does not expose bestPerChain yet. + self.assertEqual(len(df), 0) + expected_cols = { + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "leader_name", + "leader_slug", + "leader_value", + "worst_name", + "worst_slug", + "worst_value", + "schema_version", + } + self.assertEqual(set(df.columns), expected_cols) + class PartitioningTests(unittest.TestCase): def test_write_partition_path_layout(self): @@ -197,5 +308,11 @@ def test_template_substitution(self): ) +class SchemaVersionTests(unittest.TestCase): + def test_schema_version_is_v2(self): + # Guard: bumping the schema is intentional, never accidental. + self.assertEqual(SCHEMA_VERSION, 2) + + if __name__ == "__main__": unittest.main() diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index 47bcfe87..9b6d42c3 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -70,6 +70,9 @@ export async function GET( ? b.results.map((r) => ({ name: r.name, slug: r.slug, + type: r.type ?? null, + layer: r.layer ?? null, + tag: r.tag ?? null, ms: { p50: null, p90: null, p99: null, mean: null }, successRate: r.successRate, sampleSize: r.sampleSize ?? null, @@ -82,6 +85,9 @@ export async function GET( .map((r) => ({ name: r.name, slug: r.slug, + type: r.type ?? null, + layer: r.layer ?? null, + tag: r.tag ?? null, ms: r.ms, successRate: r.successRate, sampleSize: r.sampleSize, From 9e6afca91719ab25e0db036998589821a257d765 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Mon, 22 Jun 2026 17:01:27 +0200 Subject: [PATCH 2/2] fix(hf): type-narrow higher_is_better_by_slug dict + silence chain_leaders snap --- scripts/hf_publisher/publish.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/hf_publisher/publish.py b/scripts/hf_publisher/publish.py index f1800149..a19ea89e 100644 --- a/scripts/hf_publisher/publish.py +++ b/scripts/hf_publisher/publish.py @@ -290,9 +290,11 @@ def build_chain_leaders( `schemas/chain_leaders.schema.json`. """ rows: list[dict[str, Any]] = [] - # No-op iteration kept for the day the API exposes the field. - for _b in citable.get("benchmarks", []): - pass + # Reference snap.date so the param stays in the public signature + # while the body is a placeholder. Once /api/citable exposes + # bestPerChain, this block populates rows using snap.date in the + # partition column directly. + _ = (snap.date, citable.get("benchmarks", [])) # Empty DataFrame with the canonical column order so the parquet # carries its schema even when no rows are produced. columns = [ @@ -457,10 +459,10 @@ def run( # /api/citable does not surface higherIsBetter today, /api/stat does. # We backfill from the per-slug payloads so headlines carries the # field. Slugs whose stat fetch failed get a null entry. - higher_is_better_by_slug = { - s.get("slug"): bool(s.get("higherIsBetter")) + higher_is_better_by_slug: dict[str, bool] = { + slug: bool(s.get("higherIsBetter")) for s in stats - if s.get("slug") and "higherIsBetter" in s + if (slug := s.get("slug")) and "higherIsBetter" in s } headlines = build_headlines(citable, snap, higher_is_better_by_slug)