From 7ae5f19dd7e6f453161f755545649278552fd6fc Mon Sep 17 00:00:00 2001
From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com>
Date: Sat, 15 Aug 2026 17:04:39 +0200
Subject: [PATCH] add bench 208: perp DEX liquidation rate harness + spec
---
benchmarks/perp-liq-rate.yml | 197 +++++++
harnesses/perp-liq-rate/Dockerfile | 24 +
harnesses/perp-liq-rate/README.md | 62 +++
harnesses/perp-liq-rate/cmd/script/common.go | 262 ++++++++++
harnesses/perp-liq-rate/cmd/script/config.go | 102 ++++
.../perp-liq-rate/cmd/script/harness_test.go | 78 +++
.../cmd/script/integration_test.go | 53 ++
.../perp-liq-rate/cmd/script/live_test.go | 214 ++++++++
harnesses/perp-liq-rate/cmd/script/main.go | 150 ++++++
harnesses/perp-liq-rate/cmd/script/metrics.go | 110 ++++
harnesses/perp-liq-rate/cmd/script/runner.go | 97 ++++
.../perp-liq-rate/cmd/script/source_aevo.go | 63 +++
.../cmd/script/source_aevo_test.go | 77 +++
.../perp-liq-rate/cmd/script/source_dydx.go | 144 ++++++
.../cmd/script/source_dydx_test.go | 183 +++++++
.../perp-liq-rate/cmd/script/source_gains.go | 488 ++++++++++++++++++
.../cmd/script/source_gains_test.go | 351 +++++++++++++
.../perp-liq-rate/cmd/script/source_gmx.go | 125 +++++
.../cmd/script/source_gmx_test.go | 170 ++++++
.../cmd/script/source_hyperliquid.go | 286 ++++++++++
.../cmd/script/source_hyperliquid_test.go | 299 +++++++++++
.../cmd/script/source_lighter.go | 213 ++++++++
.../cmd/script/source_lighter_test.go | 308 +++++++++++
.../cmd/script/source_paradex.go | 130 +++++
.../cmd/script/source_paradex_test.go | 184 +++++++
harnesses/perp-liq-rate/cmd/script/window.go | 130 +++++
harnesses/perp-liq-rate/go.mod | 17 +
harnesses/perp-liq-rate/go.sum | 24 +
28 files changed, 4541 insertions(+)
create mode 100644 benchmarks/perp-liq-rate.yml
create mode 100644 harnesses/perp-liq-rate/Dockerfile
create mode 100644 harnesses/perp-liq-rate/README.md
create mode 100644 harnesses/perp-liq-rate/cmd/script/common.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/config.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/harness_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/integration_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/live_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/main.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/metrics.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/runner.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_aevo.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_aevo_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_dydx.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_dydx_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_gains.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_gains_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_gmx.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_gmx_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_hyperliquid.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_hyperliquid_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_lighter.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_lighter_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_paradex.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/source_paradex_test.go
create mode 100644 harnesses/perp-liq-rate/cmd/script/window.go
create mode 100644 harnesses/perp-liq-rate/go.mod
create mode 100644 harnesses/perp-liq-rate/go.sum
diff --git a/benchmarks/perp-liq-rate.yml b/benchmarks/perp-liq-rate.yml
new file mode 100644
index 00000000..8628f471
--- /dev/null
+++ b/benchmarks/perp-liq-rate.yml
@@ -0,0 +1,197 @@
+# OpenChainBench. Bench 208
+
+slug: perp-liq-rate
+number: "208"
+title: "Perp DEX liquidation rate: 24h liquidated notional as % of open interest, live"
+seo_title: "Perp DEX liquidation rate 2026"
+seo_description: "{{best_name}} shows the lowest 24h liquidation rate at {{best_p50}}% of open interest. Live comparison of liquidated notional over OI across Hyperliquid, GMX, dYdX, Lighter, gains.trade, Aevo and Paradex on ETH and BTC."
+subtitle: "Liquidated notional over the trailing 24 hours divided by current open interest, in percent. Higher means more positions are being forcibly closed relative to the venue's total exposure. Measured live every 5 minutes on ETH and BTC across 7 perpetual DEX venues."
+category: Trading
+status: live
+metric: Liquidation rate
+unit: "%"
+higher_is_better: false
+
+seo_intro: |
+ Every perpetual futures platform liquidates positions when a trader's
+ margin falls below the maintenance requirement. The liquidation rate,
+ defined as 24-hour liquidated notional divided by current open interest,
+ measures how aggressively a venue's position book turns over by force.
+ A low rate can mean tight margin parameters that leave little room for
+ error, a highly sophisticated trader base that manages risk carefully,
+ or simply low volatility during the measurement window. A high rate
+ signals that many positions are being closed at a loss, which matters
+ for traders sizing their margin buffers and for LPs trying to understand
+ adverse selection risk. This benchmark measures that rate live, every
+ five minutes, using native on-chain and API sources for both numerator
+ and denominator so the figure is apples-to-apples across venues.
+
+ The liquidation source varies by venue. Hyperliquid uses the 0xArchive
+ REST indexer, which covers all liquidation types including market-order
+ fills, giving full coverage rather than the backstop-only subset
+ accessible through the HLP vault endpoint. dYdX, Paradex and Aevo use
+ their own native REST APIs. Lighter uses Coinalyze hourly liquidation
+ buckets converted to USD at the current mark price. Gains Network on Base
+ scans TradeClosed event logs on-chain and keeps only cancelReason=1
+ (liquidation). GMX v2 open interest is read from the arbitrum-api
+ gmxinfra endpoint, giving real per-asset notional rather than a TVL
+ proxy; liquidation data remains unavailable (TheGraph subgraph defunct)
+ so GMX shows OI but a zero liquidation rate until a source is found.
+
+ Open interest is the total current notional value of all open positions
+ at each venue, sourced natively: Hyperliquid metaAndAssetCtxs, Lighter
+ orderBookDetails, dYdX REST, Paradex REST, Aevo REST, Gains Network
+ trading-variables per pair, GMX arbitrum-api markets/info. All figures
+ are in USD and use current mark price rather than entry price where a
+ live price is available.
+
+abstract: |
+ A Go exporter polls each venue every 5 minutes. Liquidation events are
+ accumulated in a 24-hour sliding window with deduplication by event key
+ (trade hash or composite). Open interest is fetched fresh each tick.
+ The liquidation rate is published as
+ perp_liq_rate_24h_pct = sum(liquidated_notional_usd, last 24h) / oi_usd * 100.
+ The first 24 hours of each venue's life in the exporter are flagged as
+ warming up (perp_liq_warming_up=1) because the sliding window is not
+ yet full. Fetch failures leave the previous gauge value unchanged and
+ increment perp_liq_fetch_errors_total; the health gauge drops to 0.
+
+methodology:
+ - "Cadence: every 5 minutes, in parallel across all venue/asset pairs."
+ - "Sliding window: liquidation events are stored in a 24-hour ring buffer keyed by a dedup string (trade hash or composite timestamp+size key). Events older than 24 hours are pruned on each tick. The window sum is the numerator of the liq rate."
+ - "Hyperliquid: OI from POST /info {type:metaAndAssetCtxs} (openInterest in base units × midPx). Liquidations from 0xArchive GET /v1/hyperliquid/liquidations/{coin} with cursor pagination, covering all liquidation types. Fallback without OXARCHIVE_API_KEY: userFillsByTime on HLP vault address (backstop-only subset)."
+ - "dYdX v4: OI and liquidations from native indexer REST. Liquidation events use the dYdX subaccount liquidation endpoint."
+ - "GMX v2 (Arbitrum): OI from arbitrum-api.gmxinfra.io/markets/info (openInterestLong + openInterestShort per market, 30-decimal USD, summed across all listed markets for the asset). Liquidation source: defunct TheGraph subgraph, returns empty; liq_rate = 0."
+ - "Lighter: OI from GET /api/v1/orderBookDetails?filter=perp (open_interest in base units × mark_price). Liquidations from Coinalyze /v1/liquidation-history (hourly buckets, symbols 0.T=ETH 1.T=BTC, base asset units × current mark_price). Requires COINALYZE_API_KEY."
+ - "gains.trade (Base): OI from GET backend-base.gains.trade/trading-variables, sum of oiLongCollateral + oiShortCollateral for the USDC collateral at the asset's pair index (BTC=0, ETH=1), divided by 1e6. Liquidations from eth_getLogs on the Gains diamond (0x6cd5ac19...) filtering TradeClosed events with cancelReason=1; notional = collateralAmount/1e6 × leverage/1e3."
+ - "Aevo: OI from native REST API. Liquidation source: no public API found, returns empty."
+ - "Paradex: OI and liquidations from native Starknet REST API."
+ - "Apple-to-apple: OI = current notional USD (collateral × leverage) at all venues. Liquidation events = notional USD of forcibly closed positions. Both sides use the same definition across all venues with available data."
+ - "GMX caveat: liq_rate is effectively 0% due to missing liquidation data; OI is real and per-asset. Do not compare GMX liq_rate to other venues until a working source is added."
+
+findings:
+ - "{{best_name}} records the lowest 24h liquidation rate at {{best_p50}}% of open interest (24h average, ETH and BTC combined)."
+ - "{{name:hyperliquid}} shows {{p50:hyperliquid}}% liq rate on a combined ETH+BTC open interest of over $4B. Hyperliquid dominates perpetual DEX volume and its liquidation coverage is the most complete of any venue here, sourced from 0xArchive which captures all liquidation types."
+ - "{{name:lighter}} shows {{p50:lighter}}% liq rate. Liquidation data via Coinalyze is available in hourly buckets; the USD conversion uses the current mark price rather than the exact price at liquidation time, so intra-hour values are approximate."
+ - "{{name:gains}} shows {{p50:gains}}% liq rate on a small Base deployment ($55K ETH OI, $161K BTC OI). Zero liquidations in quiet 24-hour windows is consistent with this OI level."
+ - "GMX liq_rate shows 0% because the GMX V2 TheGraph subgraph is defunct and no alternative public liquidation source has been identified. OI is correct ($29M ETH, $21M BTC from the gmxinfra markets/info endpoint)."
+
+source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/perp-liq-rate
+
+prometheus:
+ window: 24h
+ expected_freshness_seconds: 1800
+ freshness_metric: perp_liq_last_refresh_timestamp_seconds
+
+faq:
+ - q: "What is the liquidation rate and why does it matter?"
+ a: "The liquidation rate is the fraction of a venue's open interest that gets forcibly closed in 24 hours. A high rate means many traders are hitting their margin limits and being liquidated by the venue's insurance or backstop system. For a trader it signals how aggressively the venue's risk engine operates and whether the margin buffer they need to survive a volatile session is larger or smaller than on a competing venue."
+ - q: "Why is GMX liq rate 0%?"
+ a: "GMX v2 open interest is fetched correctly from the gmxinfra /markets/info endpoint ($29M ETH, $21M BTC). The liquidation numerator is 0 because the TheGraph subgraph used to read GMX V2 liquidation events is defunct and no alternative public API was found. The 0% figure is a data gap, not a claim that GMX has zero liquidations. OI can still be compared across venues."
+ - q: "How is Lighter liquidation data obtained?"
+ a: "Lighter's /api/v1/trades endpoint requires authentication, so direct liquidation data is not available. The bench uses Coinalyze /v1/liquidation-history for Lighter's exchange ID (symbols 0.T=ETH, 1.T=BTC), which provides hourly buckets of long and short liquidation volume in base asset units. These are converted to USD using the current mark price from orderBookDetails at the time of each 5-minute tick. The USD value of old buckets may be slightly off if the price moved significantly during the hour, but the error is bounded."
+ - q: "How is Hyperliquid liquidation data obtained?"
+ a: "With the OXARCHIVE_API_KEY environment variable set, the bench uses the 0xArchive REST API /v1/hyperliquid/liquidations/{coin} with cursor pagination, which covers all liquidation types including positions closed via market orders (not just backstop vault liquidations). Without the key the bench falls back to userFillsByTime on the HLP liquidator vault address, which captures backstop liquidations only (a minority of volume)."
+ - q: "What does the warming up period mean?"
+ a: "The first 24 hours after the harness starts, the sliding window is not yet full. The liq_rate during this period underestimates the true 24-hour rate because fewer than 24 hours of liquidation events have been accumulated. The perp_liq_warming_up metric is 1 during this period so the UI can flag the value as provisional."
+ - q: "Why are gains.trade OI values so small compared to other venues?"
+ a: "The Gains Network deployment on Base is a newer, smaller deployment than their Arbitrum and Polygon deployments. The $55K ETH OI and $161K BTC OI reported here reflect actual trading-variables from the Base chain, not the venue's total cross-chain OI. The liq_rate for Gains Base should be interpreted in this context: 0 liquidations on $55K OI is a different regime than 0 liquidations on $1B OI."
+ - q: "Is this an apples-to-apples comparison across all venues?"
+ a: "Yes for OI and liquidation definition: both sides use notional USD (position size including leverage) at all venues with available data. Hyperliquid, Lighter, dYdX and Paradex use current mark price for OI; Gains uses entry-price notional from the protocol's internal accounting (oiLongCollateral), which may differ from current-price notional if ETH/BTC prices have moved significantly since positions were opened. The main non-comparability is GMX with 0 liquidation data and Aevo/Paradex with typically low liquidation counts. The OI numbers are comparable across all venues."
+
+dimensions:
+ chain:
+ - { value: ETH, label: ETH }
+ - { value: BTC, label: BTC }
+
+providers:
+ - slug: hyperliquid
+ name: Hyperliquid
+ tag: HyperBFT L1, full liquidation coverage via 0xArchive
+ formula: "24h sum of liquidated notional USD (0xArchive /v1/hyperliquid/liquidations, all types) divided by current OI (openInterest × midPx from metaAndAssetCtxs), as a percentage. Sliding window with event-key deduplication."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="hyperliquid"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="hyperliquid"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="hyperliquid"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="hyperliquid"}[24h])
+ success: avg_over_time(perp_liq_health{venue="hyperliquid"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="hyperliquid"}[24h])
+ series: perp_liq_rate_24h_pct{venue="hyperliquid"}
+
+ - slug: dydx
+ name: dYdX v4
+ tag: Cosmos appchain perps, native liquidation API
+ formula: "24h sum of liquidated notional USD (dYdX indexer liquidation endpoint) divided by current OI (native REST), as a percentage."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="dydx"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="dydx"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="dydx"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="dydx"}[24h])
+ success: avg_over_time(perp_liq_health{venue="dydx"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="dydx"}[24h])
+ series: perp_liq_rate_24h_pct{venue="dydx"}
+
+ - slug: gmx
+ name: GMX v2
+ tag: Arbitrum synthetics, OI from gmxinfra API, liq data unavailable
+ formula: "Liq rate = 0 (TheGraph subgraph defunct, no alternative source). OI from arbitrum-api.gmxinfra.io/markets/info, sum of openInterestLong + openInterestShort across all listed markets for the asset, divided by 1e30."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="gmx"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="gmx"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="gmx"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="gmx"}[24h])
+ success: avg_over_time(perp_liq_health{venue="gmx"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="gmx"}[24h])
+ series: perp_liq_rate_24h_pct{venue="gmx"}
+
+ - slug: lighter
+ name: Lighter
+ tag: zk-rollup, liquidations via Coinalyze hourly buckets
+ formula: "24h sum of (long_liqs + short_liqs) × current_mark_price (Coinalyze /v1/liquidation-history, 1-hour resolution) divided by current OI (open_interest × mark_price from orderBookDetails), as a percentage."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="lighter"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="lighter"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="lighter"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="lighter"}[24h])
+ success: avg_over_time(perp_liq_health{venue="lighter"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="lighter"}[24h])
+ series: perp_liq_rate_24h_pct{venue="lighter"}
+
+ - slug: gains
+ name: gains.trade
+ tag: Synthetic perps on Base, liquidations from TradeClosed on-chain logs
+ formula: "24h sum of liquidated notional (collateralAmount/1e6 × leverage/1e3 from TradeClosed logs with cancelReason=1 on Base) divided by per-pair OI (oiLongCollateral + oiShortCollateral from trading-variables / 1e6, USDC collateral only), as a percentage."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="gains"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="gains"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="gains"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="gains"}[24h])
+ success: avg_over_time(perp_liq_health{venue="gains"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="gains"}[24h])
+ series: perp_liq_rate_24h_pct{venue="gains"}
+
+ - slug: aevo
+ name: Aevo
+ tag: Optimism rollup perps, no public liquidation API
+ formula: "OI from native Aevo REST API. Liquidation source: no public API found, liq_rate = 0."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="aevo"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="aevo"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="aevo"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="aevo"}[24h])
+ success: avg_over_time(perp_liq_health{venue="aevo"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="aevo"}[24h])
+ series: perp_liq_rate_24h_pct{venue="aevo"}
+
+ - slug: paradex
+ name: Paradex
+ tag: Starknet appchain perps, native liquidation API
+ formula: "24h sum of liquidated notional USD (Paradex native REST) divided by current OI (native REST), as a percentage."
+ queries:
+ p50: avg_over_time(perp_liq_rate_24h_pct{venue="paradex"}[24h])
+ p90: quantile_over_time(0.90, perp_liq_rate_24h_pct{venue="paradex"}[24h])
+ p99: quantile_over_time(0.99, perp_liq_rate_24h_pct{venue="paradex"}[24h])
+ mean: avg_over_time(perp_liq_rate_24h_pct{venue="paradex"}[24h])
+ success: avg_over_time(perp_liq_health{venue="paradex"}[24h])
+ sample_size: count_over_time(perp_liq_rate_24h_pct{venue="paradex"}[24h])
+ series: perp_liq_rate_24h_pct{venue="paradex"}
diff --git a/harnesses/perp-liq-rate/Dockerfile b/harnesses/perp-liq-rate/Dockerfile
new file mode 100644
index 00000000..52e3c42a
--- /dev/null
+++ b/harnesses/perp-liq-rate/Dockerfile
@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+
+FROM golang:1.24-alpine AS builder
+
+WORKDIR /src
+
+# -mod=mod lets the build materialize go.sum on first build if it is absent.
+ENV CGO_ENABLED=0
+
+COPY go.mod ./
+RUN go mod download
+
+COPY . .
+RUN go build -trimpath -ldflags="-s -w" -o /out/perp-liq-rate ./cmd/script
+
+FROM alpine:3.20
+
+RUN apk add --no-cache ca-certificates && adduser -D -H perpliq
+USER perpliq
+
+COPY --from=builder /out/perp-liq-rate /usr/local/bin/perp-liq-rate
+
+EXPOSE 2112
+ENTRYPOINT ["/usr/local/bin/perp-liq-rate"]
diff --git a/harnesses/perp-liq-rate/README.md b/harnesses/perp-liq-rate/README.md
new file mode 100644
index 00000000..3d438b11
--- /dev/null
+++ b/harnesses/perp-liq-rate/README.md
@@ -0,0 +1,62 @@
+# perp-liq-rate
+
+Prometheus exporter measuring `liquidated_notional_usd_24h / open_interest_usd * 100` per perpetual DEX venue per asset (ETH, BTC, SOL where available). Polls every 5 minutes, serves gauges on `:2112/metrics`.
+
+## Venues
+
+hyperliquid (ETH/BTC/SOL) · gains (ETH/BTC, Base RPC) · dydx (ETH/BTC/SOL) · gmx (ETH/BTC) · lighter (ETH/BTC) · vertex (ETH/BTC) · aevo (ETH/BTC) · paradex (ETH/BTC)
+
+## Run
+
+```bash
+go mod tidy # first checkout only: materializes go.sum
+go run ./cmd/script
+```
+
+or
+
+```bash
+docker build -t perp-liq-rate .
+docker run -p 2112:2112 perp-liq-rate
+```
+
+## Configuration
+
+| Env | Default | Meaning |
+|---|---|---|
+| `TICK_INTERVAL_SECONDS` | `300` | poll interval |
+| `RPC_BASE` | `https://mainnet.base.org` | Base mainnet JSON-RPC (gains) |
+| `LISTEN_ADDR` | `:2112` | metrics listen address |
+
+## Metrics
+
+```
+perp_liq_rate_24h_pct{venue,asset}
+perp_liq_volume_24h_usd{venue,asset}
+perp_liq_open_interest_usd{venue,asset}
+perp_liq_warming_up{venue}
+perp_liq_health{venue}
+perp_liq_last_refresh_timestamp_seconds{venue}
+perp_liq_fetch_errors_total{venue,asset,error_type}
+```
+
+`error_type` values: `http_4xx`, `http_5xx`, `http_status`, `timeout`, `decode`, `parse`, `unavailable`, `oi_zero`, `other`.
+
+## Semantics
+
+- Each (venue, asset) pair keeps a thread-safe in-memory sliding window of `(unix_ms, notional_usd)` events plus a dedup key set; both are pruned to 24h every tick.
+- All pairs are polled in parallel goroutines per tick behind a `sync.WaitGroup`.
+- On fetch error the previously published gauges are kept, `perp_liq_fetch_errors_total` is incremented and the error is logged to stdout; `perp_liq_health{venue}` drops to 0 for the tick.
+- `perp_liq_warming_up{venue}` stays 1 until 24h have elapsed since the venue's first tick. Venues with historical endpoints (gains, dydx, gmx, vertex, paradex, and partially lighter) backfill up to 24h on the first tick; hyperliquid/aevo start from their recent-trade depth.
+- lighter: HTTP 404/501 marks the venue unavailable (health 0). After 3 consecutive unavailable ticks it logs once and suppresses further error increments/logs until recovery.
+- gains: liquidations are decoded from `TradeClosed` logs on the Base diamond (`cancelReason == 1`); a Keccak-256 implementation is embedded (only external dependency allowed is the Prometheus client) and is covered by known-vector tests in `cmd/script/harness_test.go`. Event timestamps are approximated from block distance at ~2 s/block. OI uses DefiLlama Base TVL as a venue-level proxy shared by both assets.
+
+## VERIFY inventory
+
+The upstream API shapes were implemented from the written spec plus the most likely live shapes; every assumption is marked `// VERIFY:` at the exact line. Grep for them before trusting production numbers:
+
+```bash
+grep -rn "VERIFY" cmd/script
+```
+
+Highlights: the gains `TradeClosed` tuple word offsets (pairIndex / leverage / collateralAmount) and pair indices; hyperliquid's `liquidation` marker field on `recentTrades`; the dydx `perpetualMarkets` map-vs-array shape; gmx subgraph `market` field and gmxinfra `/markets` field names; lighter `/liquidations` params, envelope and market ids; vertex query path, product ids, row/timestamp fields and `max_time` cursor semantics; aevo `/liquidations` existence and timestamp encoding; paradex fills pagination cursor, auth and `open_interest` units.
diff --git a/harnesses/perp-liq-rate/cmd/script/common.go b/harnesses/perp-liq-rate/cmd/script/common.go
new file mode 100644
index 00000000..a78451e5
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/common.go
@@ -0,0 +1,262 @@
+package main
+
+// common.go — shared types and helpers used by every venue source.
+//
+// This file exists so the eight source_*.go files do not each re-implement
+// HTTP plumbing and decimal parsing. It defines the LiqEvent normalization
+// type and the Source interface that every venue implements.
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math/big"
+ "net"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// LiqEvent is a single liquidation event normalized across venues.
+type LiqEvent struct {
+ Key string // dedup key (trade hash or tx+index composite)
+ NotionalUSD float64 // liquidated notional in USD
+ TimestampMs int64 // event time, unix milliseconds
+}
+
+// Source is implemented by every venue in the source_*.go files.
+// Each venue exports exactly these two functions as methods (free functions
+// with identical names would collide inside a single package).
+type Source interface {
+ FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error)
+ FetchOI(asset string) (float64, error)
+}
+
+// ErrVenueUnavailable marks a venue as temporarily unavailable for this tick
+// (e.g. lighter returning 404/501). The runner sets health=0 but treats it
+// differently from a hard fetch error.
+var ErrVenueUnavailable = errors.New("venue unavailable")
+
+// unavailableError wraps ErrVenueUnavailable with suppression state so that
+// after N consecutive failures the runner stops incrementing error counters
+// and logging (see source_lighter.go).
+type unavailableError struct {
+ status int
+ suppressed bool
+}
+
+func (e *unavailableError) Error() string {
+ return fmt.Sprintf("venue unavailable (http %d)", e.status)
+}
+
+func (e *unavailableError) Unwrap() error { return ErrVenueUnavailable }
+
+// httpClient is shared by all venues. 10s timeout per call, per spec.
+var httpClient = &http.Client{Timeout: 10 * time.Second}
+
+// maxBodyBytes caps response bodies (DefiLlama protocol payloads can be
+// several MB because they embed full TVL history).
+const maxBodyBytes = 64 << 20
+
+// httpStatusError is returned for non-2xx responses.
+type httpStatusError struct {
+ Code int
+ URL string
+ Body string // truncated snippet, for logs
+}
+
+func (e *httpStatusError) Error() string {
+ if e.Body != "" {
+ return fmt.Sprintf("http %d from %s: %s", e.Code, e.URL, e.Body)
+ }
+ return fmt.Sprintf("http %d from %s", e.Code, e.URL)
+}
+
+// httpGetRaw performs a GET and returns the raw body of a 2xx response.
+func httpGetRaw(rawURL string) ([]byte, error) {
+ req, err := http.NewRequest(http.MethodGet, rawURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("build request: %w", err)
+ }
+ return doRaw(req)
+}
+
+// httpGetJSON performs a GET and decodes the JSON body into out.
+func httpGetJSON(rawURL string, out any) error {
+ body, err := httpGetRaw(rawURL)
+ if err != nil {
+ return err
+ }
+ return decodeJSON(body, rawURL, out)
+}
+
+// httpPostJSON performs a POST with a JSON payload and decodes the JSON
+// response into out (out may be nil to discard the body).
+func httpPostJSON(rawURL string, payload any, out any) error {
+ buf, err := json.Marshal(payload)
+ if err != nil {
+ return fmt.Errorf("marshal payload: %w", err)
+ }
+ req, err := http.NewRequest(http.MethodPost, rawURL, bytes.NewReader(buf))
+ if err != nil {
+ return fmt.Errorf("build request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ body, err := doRaw(req)
+ if err != nil {
+ return err
+ }
+ if out == nil {
+ return nil
+ }
+ return decodeJSON(body, rawURL, out)
+}
+
+func doRaw(req *http.Request) ([]byte, error) {
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("User-Agent", "perp-liq-rate/1.0")
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("http: %w", err)
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
+ if err != nil {
+ return nil, fmt.Errorf("read body: %w", err)
+ }
+ if resp.StatusCode < 200 || resp.StatusCode > 299 {
+ snippet := strings.TrimSpace(string(body))
+ if len(snippet) > 200 {
+ snippet = snippet[:200]
+ }
+ return nil, &httpStatusError{Code: resp.StatusCode, URL: req.URL.String(), Body: snippet}
+ }
+ return body, nil
+}
+
+func decodeJSON(body []byte, rawURL string, out any) error {
+ if err := json.Unmarshal(body, out); err != nil {
+ host := rawURL
+ if u, uerr := url.Parse(rawURL); uerr == nil {
+ host = u.Host
+ }
+ return fmt.Errorf("decode response from %s: %w", host, err)
+ }
+ return nil
+}
+
+// decodeListFlexible decodes either a bare JSON array or an object wrapping
+// the array under the given key (some venues are inconsistent about this).
+func decodeListFlexible(body []byte, key string, out any) error {
+ trimmed := bytes.TrimSpace(body)
+ if len(trimmed) > 0 && trimmed[0] == '[' {
+ return json.Unmarshal(trimmed, out)
+ }
+ var m map[string]json.RawMessage
+ if err := json.Unmarshal(trimmed, &m); err != nil {
+ return fmt.Errorf("decode envelope: %w", err)
+ }
+ v, ok := m[key]
+ if !ok {
+ return fmt.Errorf("response missing %q array", key)
+ }
+ return json.Unmarshal(v, out)
+}
+
+// parseF parses a plain decimal float string.
+func parseF(s string) (float64, error) {
+ v, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
+ if err != nil {
+ return 0, fmt.Errorf("parse float %q: %w", s, err)
+ }
+ return v, nil
+}
+
+// parseScaled parses a (possibly huge) decimal integer string and divides it
+// by 10^decimals, using big.Float so 30-decimal fixed-point values (GMX) and
+// i128 strings (Vertex) do not overflow along the way.
+func parseScaled(s string, decimals int) (float64, error) {
+ s = strings.TrimSpace(s)
+ if s == "" {
+ return 0, fmt.Errorf("empty numeric string")
+ }
+ f, ok := new(big.Float).SetPrec(256).SetString(s)
+ if !ok {
+ return 0, fmt.Errorf("bad numeric string %q", s)
+ }
+ if decimals > 0 {
+ scale := new(big.Float).SetPrec(256).SetInt(new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(decimals)), nil))
+ f.Quo(f, scale)
+ }
+ v, _ := f.Float64()
+ return v, nil
+}
+
+// flexFloat unmarshals JSON values that may arrive either as a number or as
+// a numeric string ("123.4" vs 123.4). Null decodes to 0.
+type flexFloat float64
+
+func (f *flexFloat) UnmarshalJSON(b []byte) error {
+ b = bytes.TrimSpace(b)
+ if len(b) == 0 || string(b) == "null" {
+ *f = 0
+ return nil
+ }
+ if b[0] == '"' {
+ var s string
+ if err := json.Unmarshal(b, &s); err != nil {
+ return err
+ }
+ s = strings.TrimSpace(s)
+ if s == "" {
+ *f = 0
+ return nil
+ }
+ v, err := strconv.ParseFloat(s, 64)
+ if err != nil {
+ return err
+ }
+ *f = flexFloat(v)
+ return nil
+ }
+ var v float64
+ if err := json.Unmarshal(b, &v); err != nil {
+ return err
+ }
+ *f = flexFloat(v)
+ return nil
+}
+
+// classifyError maps an error to a low-cardinality error_type label value.
+func classifyError(err error) string {
+ var statusErr *httpStatusError
+ var jsonSyn *json.SyntaxError
+ var jsonType *json.UnmarshalTypeError
+ var numErr *strconv.NumError
+ var netErr net.Error
+ switch {
+ case errors.Is(err, ErrVenueUnavailable):
+ return "unavailable"
+ case errors.As(err, &statusErr):
+ switch {
+ case statusErr.Code >= 500:
+ return "http_5xx"
+ case statusErr.Code >= 400:
+ return "http_4xx"
+ default:
+ return "http_status"
+ }
+ case errors.As(err, &netErr) && netErr.Timeout():
+ return "timeout"
+ case errors.As(err, &jsonSyn), errors.As(err, &jsonType):
+ return "decode"
+ case errors.As(err, &numErr):
+ return "parse"
+ default:
+ return "other"
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/config.go b/harnesses/perp-liq-rate/cmd/script/config.go
new file mode 100644
index 00000000..ce37fc4c
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/config.go
@@ -0,0 +1,102 @@
+package main
+
+// config.go — environment handling and the VenueAsset registry that binds
+// every (venue, asset) pair to its Source implementation.
+
+import (
+ "fmt"
+ "os"
+ "strconv"
+ "time"
+)
+
+// VenueAsset binds one venue+asset pair to the Source that serves it.
+// Several pairs may share a single Source instance (per-venue state such as
+// the gains block cursor lives on that instance).
+type VenueAsset struct {
+ Venue string
+ Asset string
+ Source Source
+}
+
+// Config is the fully resolved runtime configuration.
+type Config struct {
+ TickInterval time.Duration
+ ListenAddr string
+ RPCBase string
+ Pairs []VenueAsset
+}
+
+const (
+ defaultTickSeconds = 300
+ defaultListenAddr = ":2112"
+ defaultRPCBase = "https://mainnet.base.org"
+)
+
+// loadConfig reads environment variables and builds the venue registry.
+//
+// Environment:
+//
+// TICK_INTERVAL_SECONDS — poll interval, default 300
+// RPC_BASE — Base mainnet JSON-RPC URL, default https://mainnet.base.org
+// LISTEN_ADDR — metrics listen address, default :2112
+func loadConfig() (*Config, error) {
+ tickSeconds := defaultTickSeconds
+ if v := os.Getenv("TICK_INTERVAL_SECONDS"); v != "" {
+ n, err := strconv.Atoi(v)
+ if err != nil || n <= 0 {
+ return nil, fmt.Errorf("invalid TICK_INTERVAL_SECONDS %q", v)
+ }
+ tickSeconds = n
+ }
+
+ rpcBase := os.Getenv("RPC_BASE")
+ if rpcBase == "" {
+ rpcBase = defaultRPCBase
+ }
+
+ listen := os.Getenv("LISTEN_ADDR")
+ if listen == "" {
+ listen = defaultListenAddr
+ }
+
+ hyperliquid := NewHyperliquid()
+ gains := NewGains(rpcBase)
+ dydx := NewDydx()
+ gmx := NewGMX()
+ lighter := NewLighter()
+ aevo := NewAevo()
+ paradex := NewParadex()
+
+ pairs := []VenueAsset{
+ {Venue: "hyperliquid", Asset: "ETH", Source: hyperliquid},
+ {Venue: "hyperliquid", Asset: "BTC", Source: hyperliquid},
+ {Venue: "hyperliquid", Asset: "SOL", Source: hyperliquid},
+
+ {Venue: "gains", Asset: "ETH", Source: gains},
+ {Venue: "gains", Asset: "BTC", Source: gains},
+
+ {Venue: "dydx", Asset: "ETH", Source: dydx},
+ {Venue: "dydx", Asset: "BTC", Source: dydx},
+ {Venue: "dydx", Asset: "SOL", Source: dydx},
+
+ {Venue: "gmx", Asset: "ETH", Source: gmx},
+ {Venue: "gmx", Asset: "BTC", Source: gmx},
+
+ {Venue: "lighter", Asset: "ETH", Source: lighter},
+ {Venue: "lighter", Asset: "BTC", Source: lighter},
+
+ {Venue: "aevo", Asset: "ETH", Source: aevo},
+ {Venue: "aevo", Asset: "BTC", Source: aevo},
+
+ {Venue: "paradex", Asset: "ETH", Source: paradex},
+ {Venue: "paradex", Asset: "BTC", Source: paradex},
+ }
+
+ return &Config{
+ TickInterval: time.Duration(tickSeconds) * time.Second,
+ ListenAddr: listen,
+ RPCBase: rpcBase,
+ Pairs: pairs,
+ }, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/harness_test.go b/harnesses/perp-liq-rate/cmd/script/harness_test.go
new file mode 100644
index 00000000..5426fad2
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/harness_test.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "encoding/hex"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestKeccak256KnownVectors(t *testing.T) {
+ cases := []struct {
+ in string
+ want string
+ }{
+ {"", "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"},
+ {"Transfer(address,address,uint256)", "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"},
+ // > one rate block (136 bytes) to exercise multi-block absorption
+ {strings.Repeat("a", 200), ""},
+ }
+ for _, c := range cases {
+ h := keccak256([]byte(c.in))
+ got := hex.EncodeToString(h[:])
+ if c.want != "" && got != c.want {
+ t.Fatalf("keccak256(%q) = %s, want %s", c.in, got, c.want)
+ }
+ t.Logf("keccak256(len=%d) = %s", len(c.in), got)
+ }
+}
+
+func TestGainsTopicNonEmpty(t *testing.T) {
+ if len(gainsTradeClosedTopic) != 66 || !strings.HasPrefix(gainsTradeClosedTopic, "0x") {
+ t.Fatalf("bad topic %q", gainsTradeClosedTopic)
+ }
+ t.Logf("TradeClosed topic0 = %s", gainsTradeClosedTopic)
+}
+
+func TestSlidingWindowAndSeenSet(t *testing.T) {
+ w := NewSlidingWindow(24 * time.Hour)
+ now := time.Now().UnixMilli()
+ w.Add(now-25*3600*1000, 100) // stale
+ w.Add(now-3600*1000, 50)
+ w.Add(now, 25)
+ w.Prune(now)
+ if got := w.Sum(); got != 75 {
+ t.Fatalf("Sum = %v, want 75", got)
+ }
+ if w.Len() != 2 {
+ t.Fatalf("Len = %d, want 2", w.Len())
+ }
+ if w.IsWarm(time.Now()) {
+ t.Fatalf("window should not be warm before MarkTick+span")
+ }
+ w.MarkTick(time.Now().Add(-25 * time.Hour))
+ if !w.IsWarm(time.Now()) {
+ t.Fatalf("window should be warm 25h after first tick")
+ }
+
+ s := NewSeenSet()
+ if !s.Add("k1", now) || s.Add("k1", now) {
+ t.Fatalf("SeenSet dedup broken")
+ }
+ s.Add("old", now-25*3600*1000)
+ s.Prune(now - 24*3600*1000)
+ if s.Len() != 1 {
+ t.Fatalf("SeenSet prune broken, len=%d", s.Len())
+ }
+}
+
+func TestParseScaled(t *testing.T) {
+ v, err := parseScaled("1230000000000000000000000000000000", 30) // 1230 with 30 dp
+ if err != nil || v < 1229.999 || v > 1230.001 {
+ t.Fatalf("parseScaled 30dp = %v (%v)", v, err)
+ }
+ v, err = parseScaled("-2500000000000000000", 18)
+ if err != nil || v != -2.5 {
+ t.Fatalf("parseScaled negative = %v (%v)", v, err)
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/integration_test.go b/harnesses/perp-liq-rate/cmd/script/integration_test.go
new file mode 100644
index 00000000..ea1b582d
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/integration_test.go
@@ -0,0 +1,53 @@
+//go:build integration
+
+package main
+
+import (
+ "testing"
+)
+
+func TestIntegration_Hyperliquid(t *testing.T) {
+ t.Skip("requires -tags integration")
+ h := NewHyperliquid()
+ oi, err := h.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("Hyperliquid FetchOI: %v", err)
+ }
+ if oi <= 0 {
+ t.Errorf("expected positive OI, got %v", oi)
+ }
+ t.Logf("Hyperliquid ETH OI = %.2f", oi)
+}
+
+func TestIntegration_Dydx(t *testing.T) {
+ t.Skip("requires -tags integration")
+ d := NewDydx()
+ oi, err := d.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("dYdX FetchOI: %v", err)
+ }
+ if oi <= 0 {
+ t.Errorf("expected positive OI, got %v", oi)
+ }
+ t.Logf("dYdX ETH OI = %.2f", oi)
+}
+
+func TestIntegration_GMX(t *testing.T) {
+ t.Skip("requires -tags integration")
+ g := NewGMX()
+ markets, err := g.fetchMarkets()
+ if err != nil {
+ t.Fatalf("GMX fetchMarkets: %v", err)
+ }
+ found := false
+ for _, m := range markets {
+ if gmxAssetMatches(m.Name, "ETH") {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Error("no ETH market found in GMX /markets")
+ }
+ t.Logf("GMX markets fetched: %d total", len(markets))
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/live_test.go b/harnesses/perp-liq-rate/cmd/script/live_test.go
new file mode 100644
index 00000000..5512a050
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/live_test.go
@@ -0,0 +1,214 @@
+//go:build live
+
+package main
+
+import (
+ "fmt"
+ "testing"
+ "time"
+)
+
+// Run with: go test -tags live -v -timeout 120s -run TestLive
+
+func TestLive_AllVenuesOI(t *testing.T) {
+ type result struct {
+ venue string
+ asset string
+ oi float64
+ err error
+ }
+
+ type venueSource struct {
+ name string
+ source Source
+ }
+
+ sources := []venueSource{
+ {"hyperliquid", NewHyperliquid()},
+ {"dydx", NewDydx()},
+ {"gmx", NewGMX()},
+ {"lighter", NewLighter()},
+ {"gains", NewGains("https://mainnet.base.org")},
+ {"aevo", NewAevo()},
+ {"paradex", NewParadex()},
+ }
+ assets := []string{"ETH", "BTC"}
+
+ results := make([]result, 0, len(sources)*len(assets))
+ for _, vs := range sources {
+ for _, asset := range assets {
+ oi, err := vs.source.FetchOI(asset)
+ results = append(results, result{vs.name, asset, oi, err})
+ }
+ }
+
+ fmt.Println("\n=== OI Results (USD) ===")
+ fmt.Printf("%-15s %-5s %20s %s\n", "VENUE", "ASSET", "OI_USD", "STATUS")
+ fmt.Println("-------------------------------------------------------------")
+ for _, r := range results {
+ if r.err != nil {
+ fmt.Printf("%-15s %-5s %20s ERROR: %v\n", r.venue, r.asset, "-", r.err)
+ } else {
+ fmt.Printf("%-15s %-5s %20.2f OK\n", r.venue, r.asset, r.oi)
+ }
+ }
+}
+
+func TestLive_AllVenuesLiquidations(t *testing.T) {
+ type result struct {
+ venue string
+ asset string
+ count int
+ volume float64
+ err error
+ }
+
+ type venueSource struct {
+ name string
+ source Source
+ }
+
+ sinceMs := time.Now().Add(-24 * time.Hour).UnixMilli()
+
+ sources := []venueSource{
+ {"hyperliquid", NewHyperliquid()},
+ {"dydx", NewDydx()},
+ {"gmx", NewGMX()},
+ {"lighter", NewLighter()},
+ {"gains", NewGains("https://mainnet.base.org")},
+ {"aevo", NewAevo()},
+ {"paradex", NewParadex()},
+ }
+ assets := []string{"ETH", "BTC"}
+
+ results := make([]result, 0, len(sources)*len(assets))
+ for _, vs := range sources {
+ for _, asset := range assets {
+ events, err := vs.source.FetchLiquidationsSince(asset, sinceMs)
+ vol := 0.0
+ for _, e := range events {
+ vol += e.NotionalUSD
+ }
+ results = append(results, result{vs.name, asset, len(events), vol, err})
+ }
+ }
+
+ fmt.Println("\n=== Liquidations (last 24h, first tick only) ===")
+ fmt.Printf("%-15s %-5s %8s %20s %s\n", "VENUE", "ASSET", "COUNT", "VOLUME_USD", "STATUS")
+ fmt.Println("-----------------------------------------------------------------------")
+ for _, r := range results {
+ if r.err != nil {
+ fmt.Printf("%-15s %-5s %8s %20s ERROR: %v\n", r.venue, r.asset, "-", "-", r.err)
+ } else {
+ fmt.Printf("%-15s %-5s %8d %20.2f OK\n", r.venue, r.asset, r.count, r.volume)
+ }
+ }
+}
+
+func TestLive_LighterEndpoints(t *testing.T) {
+ l := NewLighter()
+
+ fmt.Println("\n=== Lighter: orderBookDetails ===")
+ markets, err := l.fetchMarkets()
+ if err != nil {
+ t.Fatalf("fetchMarkets: %v", err)
+ }
+ for _, m := range markets {
+ fmt.Printf(" symbol=%-12s market_id=%d OI=%.4f mark_price=%.4f OI_USD=%.2f\n",
+ m.Symbol, m.MarketID, m.OpenInterest, float64(m.MarkPrice),
+ m.OpenInterest*float64(m.MarkPrice))
+ }
+
+ fmt.Println("\n=== Lighter: ETH liquidations (last 1h) ===")
+ sinceMs := time.Now().Add(-1 * time.Hour).UnixMilli()
+ events, err := l.FetchLiquidationsSince("ETH", sinceMs)
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ } else {
+ fmt.Printf(" %d events in last 1h\n", len(events))
+ for i, e := range events {
+ if i >= 5 {
+ fmt.Printf(" ... and %d more\n", len(events)-5)
+ break
+ }
+ fmt.Printf(" key=%s notional=%.2f ts=%d\n", e.Key, e.NotionalUSD, e.TimestampMs)
+ }
+ }
+}
+
+func TestLive_ParadexEndpoints(t *testing.T) {
+ p := NewParadex()
+
+ fmt.Println("\n=== Paradex: ETH OI ===")
+ oi, err := p.FetchOI("ETH")
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ } else {
+ fmt.Printf(" OI = %.2f\n", oi)
+ }
+
+ fmt.Println("\n=== Paradex: ETH liquidations (last 1h) ===")
+ sinceMs := time.Now().Add(-1 * time.Hour).UnixMilli()
+ events, err := p.FetchLiquidationsSince("ETH", sinceMs)
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ } else {
+ fmt.Printf(" %d events in last 1h\n", len(events))
+ for i, e := range events {
+ if i >= 5 {
+ fmt.Printf(" ... and %d more\n", len(events)-5)
+ break
+ }
+ fmt.Printf(" key=%s notional=%.2f ts=%d\n", e.Key, e.NotionalUSD, e.TimestampMs)
+ }
+ }
+}
+
+func TestLive_HyperliquidRecentTrades(t *testing.T) {
+ h := NewHyperliquid()
+ fmt.Println("\n=== Hyperliquid: recentTrades ETH (liquidation field check) ===")
+ sinceMs := time.Now().Add(-1 * time.Hour).UnixMilli()
+ events, err := h.FetchLiquidationsSince("ETH", sinceMs)
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ return
+ }
+ fmt.Printf(" %d liquidation events in last 1h from recentTrades\n", len(events))
+ if len(events) == 0 {
+ fmt.Println(" NOTE: recentTrades may not carry a liquidation flag — field always absent/null")
+ }
+
+ fmt.Println("\n=== Hyperliquid: ETH OI ===")
+ oi, err := h.FetchOI("ETH")
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ } else {
+ fmt.Printf(" OI = %.2f USD\n", oi)
+ }
+}
+
+func TestLive_DydxSanity(t *testing.T) {
+ d := NewDydx()
+ fmt.Println("\n=== dYdX: ETH liquidations (last 24h) ===")
+ sinceMs := time.Now().Add(-24 * time.Hour).UnixMilli()
+ events, err := d.FetchLiquidationsSince("ETH", sinceMs)
+ if err != nil {
+ fmt.Printf(" ERROR: %v\n", err)
+ return
+ }
+ vol := 0.0
+ for _, e := range events {
+ vol += e.NotionalUSD
+ }
+ fmt.Printf(" %d events, volume=%.2f USD\n", len(events), vol)
+ oi, err := d.FetchOI("ETH")
+ if err != nil {
+ fmt.Printf(" OI ERROR: %v\n", err)
+ } else {
+ if oi > 0 && vol > 0 {
+ fmt.Printf(" OI=%.2f liq_rate=%.4f%%\n", oi, vol/oi*100)
+ } else {
+ fmt.Printf(" OI=%.2f (no liquidations yet in window)\n", oi)
+ }
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/main.go b/harnesses/perp-liq-rate/cmd/script/main.go
new file mode 100644
index 00000000..7ed2dc3d
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/main.go
@@ -0,0 +1,150 @@
+package main
+
+// main.go — process entrypoint: signal handling, the metrics HTTP server
+// goroutine, and the tick loop that fans out per-venue-asset goroutines.
+
+import (
+ "context"
+ "log"
+ "net/http"
+ "os"
+ "os/signal"
+ "sync"
+ "syscall"
+ "time"
+)
+
+// pairRuntime carries the cross-tick state of one venue+asset pair.
+type pairRuntime struct {
+ va VenueAsset
+ window *SlidingWindow
+ seen *SeenSet
+ sinceMs int64 // high-water mark for FetchLiquidationsSince
+}
+
+func main() {
+ log.SetOutput(os.Stdout)
+ log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.LUTC)
+
+ cfg, err := loadConfig()
+ if err != nil {
+ log.Fatalf("config: %v", err)
+ }
+ log.Printf("perp-liq-rate starting: %d venue/asset pairs, tick=%s, listen=%s",
+ len(cfg.Pairs), cfg.TickInterval, cfg.ListenAddr)
+
+ reg := registerMetrics()
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", metricsHandler(reg))
+ mux.HandleFunc("/health", func(rw http.ResponseWriter, _ *http.Request) {
+ rw.WriteHeader(http.StatusOK)
+ _, _ = rw.Write([]byte("ok\n"))
+ })
+ srv := &http.Server{
+ Addr: cfg.ListenAddr,
+ Handler: mux,
+ ReadHeaderTimeout: 5 * time.Second,
+ }
+
+ ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+ defer stop()
+
+ go func() {
+ log.Printf("metrics server listening on %s", cfg.ListenAddr)
+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatalf("metrics server: %v", err)
+ }
+ }()
+
+ // Build per-pair runtime state. The initial since is now-24h so venues
+ // with historical endpoints backfill the full window on the first tick.
+ startBackfillMs := time.Now().Add(-windowSpan).UnixMilli()
+ pairs := make([]*pairRuntime, 0, len(cfg.Pairs))
+ venues := make(map[string]bool, 8)
+ for _, va := range cfg.Pairs {
+ pairs = append(pairs, &pairRuntime{
+ va: va,
+ window: NewSlidingWindow(windowSpan),
+ seen: NewSeenSet(),
+ sinceMs: startBackfillMs,
+ })
+ venues[va.Venue] = true
+ }
+
+ // Before the first tick completes, every venue is warming up.
+ for venue := range venues {
+ setVenueWarming(venue, true)
+ }
+
+ runOnce := func() {
+ tickStart := time.Now()
+ tickStartMs := tickStart.UnixMilli()
+
+ // Venue-level success aggregation across its asset goroutines.
+ venueOK := make(map[string]bool, len(venues))
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+
+ for _, p := range pairs {
+ wg.Add(1)
+ go func(p *pairRuntime) {
+ defer wg.Done()
+ ok := runTick(p.va, p.window, p.seen, p.sinceMs)
+ if ok {
+ // Next tick fetches from the start of this one; the
+ // overlap is harmless because of the SeenSet dedup.
+ p.sinceMs = tickStartMs
+ }
+ mu.Lock()
+ if prev, present := venueOK[p.va.Venue]; present {
+ venueOK[p.va.Venue] = prev && ok
+ } else {
+ venueOK[p.va.Venue] = ok
+ }
+ mu.Unlock()
+ }(p)
+ }
+ wg.Wait()
+
+ now := time.Now()
+ // Warm-up: a venue is warm once every one of its windows has seen a
+ // full 24h since its first tick (they all start together, so this is
+ // effectively "24h since the venue's first tick").
+ venueWarm := make(map[string]bool, len(venues))
+ for venue := range venues {
+ venueWarm[venue] = true
+ }
+ for _, p := range pairs {
+ if !p.window.IsWarm(now) {
+ venueWarm[p.va.Venue] = false
+ }
+ }
+ for venue, allOK := range venueOK {
+ setVenueHealth(venue, allOK)
+ if allOK {
+ setVenueRefreshed(venue, now)
+ }
+ setVenueWarming(venue, !venueWarm[venue])
+ }
+ log.Printf("tick complete in %s", time.Since(tickStart).Round(time.Millisecond))
+ }
+
+ ticker := time.NewTicker(cfg.TickInterval)
+ defer ticker.Stop()
+
+ runOnce()
+ for {
+ select {
+ case <-ctx.Done():
+ log.Printf("shutdown signal received, stopping")
+ shCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := srv.Shutdown(shCtx); err != nil {
+ log.Printf("metrics server shutdown: %v", err)
+ }
+ return
+ case <-ticker.C:
+ runOnce()
+ }
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/metrics.go b/harnesses/perp-liq-rate/cmd/script/metrics.go
new file mode 100644
index 00000000..e4bd2c0d
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/metrics.go
@@ -0,0 +1,110 @@
+package main
+
+// metrics.go — all Prometheus collectors plus small update helpers so the
+// rest of the code never touches label plumbing directly.
+
+import (
+ "net/http"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+ liqRate = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_rate_24h_pct",
+ Help: "Liquidated notional over the trailing 24h as a percentage of open interest (liq_usd / oi_usd * 100).",
+ }, []string{"venue", "chain"})
+
+ liqVolume = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_volume_24h_usd",
+ Help: "Sum of liquidated notional (USD) over the trailing 24h sliding window.",
+ }, []string{"venue", "chain"})
+
+ liqOpenInterest = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_open_interest_usd",
+ Help: "Current open interest (USD) per venue and asset (TVL proxy for venues without a native OI endpoint).",
+ }, []string{"venue", "chain"})
+
+ liqWarmingUp = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_warming_up",
+ Help: "1 until 24h have elapsed since the venue's first tick (24h sums incomplete before that), else 0.",
+ }, []string{"venue"})
+
+ liqHealth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_health",
+ Help: "1 if all fetches for the venue succeeded on the most recent tick, else 0.",
+ }, []string{"venue"})
+
+ liqLastRefresh = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+ Name: "perp_liq_last_refresh_timestamp_seconds",
+ Help: "Unix timestamp of the last fully successful tick for the venue.",
+ }, []string{"venue"})
+
+ liqFetchErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "perp_liq_fetch_errors_total",
+ Help: "Fetch/decode errors per venue, asset and error type.",
+ }, []string{"venue", "chain", "error_type"})
+)
+
+// registerMetrics builds a dedicated registry containing only this
+// exporter's collectors.
+func registerMetrics() *prometheus.Registry {
+ reg := prometheus.NewRegistry()
+ reg.MustRegister(
+ liqRate,
+ liqVolume,
+ liqOpenInterest,
+ liqWarmingUp,
+ liqHealth,
+ liqLastRefresh,
+ liqFetchErrors,
+ )
+ return reg
+}
+
+// metricsHandler returns the /metrics HTTP handler for the registry.
+func metricsHandler(reg *prometheus.Registry) http.Handler {
+ return promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
+}
+
+// setLiqVolume publishes the 24h liquidation volume for a venue/asset.
+func setLiqVolume(venue, asset string, volumeUSD float64) {
+ liqVolume.WithLabelValues(venue, asset).Set(volumeUSD)
+}
+
+// setOIAndRate publishes open interest and, given the current window sum,
+// the liquidation rate. Callers must only invoke this with oiUSD > 0.
+func setOIAndRate(venue, asset string, volumeUSD, oiUSD float64) {
+ liqOpenInterest.WithLabelValues(venue, asset).Set(oiUSD)
+ liqRate.WithLabelValues(venue, asset).Set(volumeUSD / oiUSD * 100)
+}
+
+// recordFetchError increments the error counter with a classified type.
+func recordFetchError(venue, asset, errType string) {
+ liqFetchErrors.WithLabelValues(venue, asset, errType).Inc()
+}
+
+// setVenueHealth publishes venue health (1 healthy / 0 degraded).
+func setVenueHealth(venue string, healthy bool) {
+ v := 0.0
+ if healthy {
+ v = 1.0
+ }
+ liqHealth.WithLabelValues(venue).Set(v)
+}
+
+// setVenueWarming publishes the warm-up flag for the venue.
+func setVenueWarming(venue string, warming bool) {
+ v := 0.0
+ if warming {
+ v = 1.0
+ }
+ liqWarmingUp.WithLabelValues(venue).Set(v)
+}
+
+// setVenueRefreshed stamps the last fully successful tick time.
+func setVenueRefreshed(venue string, t time.Time) {
+ liqLastRefresh.WithLabelValues(venue).Set(float64(t.Unix()))
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/runner.go b/harnesses/perp-liq-rate/cmd/script/runner.go
new file mode 100644
index 00000000..ecec03fd
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/runner.go
@@ -0,0 +1,97 @@
+package main
+
+// runner.go — runTick is executed once per (venue, asset) pair per tick, in
+// its own goroutine.
+
+import (
+ "errors"
+ "log"
+ "time"
+)
+
+// windowSpan is the sliding-window length the whole harness is built around.
+const windowSpan = 24 * time.Hour
+
+// runTick performs one poll cycle for a single venue+asset pair:
+// fetch liquidations since the last tick, dedup into the window, prune the
+// window and seen-set, fetch OI, and publish gauges.
+//
+// sinceMs is the caller-managed high-water mark (unix ms) for liquidation
+// fetches; on the first tick it is now-24h so venues with historical
+// endpoints backfill the full window. runTick reports whether both fetches
+// succeeded so the caller can advance sinceMs and aggregate venue health.
+// On any error the previously published gauges are intentionally left
+// untouched.
+func runTick(va VenueAsset, w *SlidingWindow, seen *SeenSet, sinceMs int64) bool {
+ now := time.Now()
+ nowMs := now.UnixMilli()
+ cutoffMs := nowMs - windowSpan.Milliseconds()
+ w.MarkTick(now)
+
+ ok := true
+
+ events, liqErr := va.Source.FetchLiquidationsSince(va.Asset, sinceMs)
+ if liqErr != nil {
+ handleFetchError(va, "liquidations", liqErr)
+ ok = false
+ } else {
+ added := 0
+ for _, e := range events {
+ if e.Key == "" || e.TimestampMs <= 0 || e.NotionalUSD <= 0 {
+ continue
+ }
+ if e.TimestampMs < cutoffMs {
+ continue // older than the window; irrelevant
+ }
+ if seen.Add(e.Key, e.TimestampMs) {
+ w.Add(e.TimestampMs, e.NotionalUSD)
+ added++
+ }
+ }
+ if added > 0 {
+ log.Printf("[%s/%s] +%d liquidation event(s), window now %d event(s)",
+ va.Venue, va.Asset, added, w.Len())
+ }
+ }
+
+ w.Prune(nowMs)
+ seen.Prune(cutoffMs)
+
+ oi, oiErr := va.Source.FetchOI(va.Asset)
+ if oiErr != nil {
+ handleFetchError(va, "oi", oiErr)
+ ok = false
+ }
+
+ // Publish. Volume is valid whenever the liquidation fetch succeeded; the
+ // rate additionally needs a positive OI. On failure the previous gauge
+ // values are kept as-is (spec: "on error keep previous gauge").
+ if liqErr == nil {
+ volume := w.Sum()
+ setLiqVolume(va.Venue, va.Asset, volume)
+ if oiErr == nil {
+ if oi > 0 {
+ setOIAndRate(va.Venue, va.Asset, volume, oi)
+ } else {
+ recordFetchError(va.Venue, va.Asset, "oi_zero")
+ log.Printf("[%s/%s] OI endpoint returned non-positive value %.4f; keeping previous OI/rate gauges", va.Venue, va.Asset, oi)
+ ok = false
+ }
+ }
+ }
+
+ return ok
+}
+
+// handleFetchError classifies, counts and logs a fetch error, honoring the
+// suppression flag used by venues that are marked unavailable repeatedly.
+func handleFetchError(va VenueAsset, stage string, err error) {
+ var ue *unavailableError
+ if errors.As(err, &ue) && ue.suppressed {
+ // After repeated consecutive unavailability the source asked us to
+ // stay quiet (logged once at the threshold, see source_lighter.go).
+ return
+ }
+ recordFetchError(va.Venue, va.Asset, classifyError(err))
+ log.Printf("[%s/%s] %s fetch error: %v", va.Venue, va.Asset, stage, err)
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_aevo.go b/harnesses/perp-liq-rate/cmd/script/source_aevo.go
new file mode 100644
index 00000000..1d1218c2
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_aevo.go
@@ -0,0 +1,63 @@
+package main
+
+// source_aevo.go — Aevo.
+//
+// Liquidations: Aevo has no public liquidation feed as of 2025-08;
+// FetchLiquidationsSince returns empty with no error.
+// OI: GET /statistics?asset=&instrument_type=PERPETUAL →
+// open_interest.total (contracts) * mark_price (USD/contract).
+
+import (
+ "fmt"
+ "strings"
+)
+
+const aevoBaseURL = "https://api.aevo.xyz"
+
+var aevoInstruments = map[string]string{
+ "ETH": "ETH-PERP",
+ "BTC": "BTC-PERP",
+}
+
+// Aevo implements Source.
+type Aevo struct {
+ baseURL string // defaults to aevoBaseURL
+}
+
+// NewAevo returns the Aevo source.
+func NewAevo() *Aevo { return &Aevo{baseURL: aevoBaseURL} }
+
+// FetchLiquidationsSince returns empty — Aevo has no public liquidation feed.
+func (a *Aevo) FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error) {
+ if _, ok := aevoInstruments[asset]; !ok {
+ return nil, fmt.Errorf("aevo: unsupported asset %q", asset)
+ }
+ return nil, nil
+}
+
+// FetchOI returns open interest in USD from /statistics.
+// VERIFY: response shape — open_interest.total is in contract units (ETH/BTC),
+// mark_price is the USD price; OI_USD = total * mark_price.
+func (a *Aevo) FetchOI(asset string) (float64, error) {
+ instrument, ok := aevoInstruments[asset]
+ if !ok {
+ return 0, fmt.Errorf("aevo: unsupported asset %q", asset)
+ }
+ base := strings.TrimSuffix(instrument, "-PERP")
+ var resp struct {
+ OpenInterest struct {
+ Total flexFloat `json:"total"`
+ } `json:"open_interest"`
+ MarkPrice flexFloat `json:"mark_price"`
+ }
+ u := fmt.Sprintf("%s/statistics?asset=%s&instrument_type=PERPETUAL", a.baseURL, base)
+ if err := httpGetJSON(u, &resp); err != nil {
+ return 0, fmt.Errorf("aevo statistics: %w", err)
+ }
+ total := float64(resp.OpenInterest.Total)
+ markPrice := float64(resp.MarkPrice)
+ if markPrice == 0 {
+ return 0, fmt.Errorf("aevo statistics: mark_price is zero")
+ }
+ return total * markPrice, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_aevo_test.go b/harnesses/perp-liq-rate/cmd/script/source_aevo_test.go
new file mode 100644
index 00000000..f0c04b01
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_aevo_test.go
@@ -0,0 +1,77 @@
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestAevo_FetchLiquidationsSince_ReturnsEmpty(t *testing.T) {
+ a := &Aevo{baseURL: "http://unused"}
+ events, err := a.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events, got %d", len(events))
+ }
+}
+
+func TestAevo_FetchLiquidationsSince_UnknownAsset(t *testing.T) {
+ a := &Aevo{baseURL: "http://unused"}
+ _, err := a.FetchLiquidationsSince("SOL", 0)
+ if err == nil {
+ t.Fatal("expected error for unsupported asset, got nil")
+ }
+}
+
+func TestAevo_FetchOI_HappyPath(t *testing.T) {
+ // open_interest.total is in contract units; OI_USD = total * mark_price
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasPrefix(r.URL.Path, "/statistics") {
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "open_interest": map[string]any{"total": "10.5"},
+ "mark_price": "3000.0",
+ })
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ a := &Aevo{baseURL: srv.URL}
+ oi, err := a.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // 10.5 * 3000 = 31500
+ if oi < 31499 || oi > 31501 {
+ t.Errorf("OI = %v, want ~31500", oi)
+ }
+}
+
+func TestAevo_FetchOI_ZeroPriceFails(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "open_interest": map[string]any{"total": "10.5"},
+ "mark_price": "0",
+ })
+ }))
+ defer srv.Close()
+
+ a := &Aevo{baseURL: srv.URL}
+ _, err := a.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for zero mark_price, got nil")
+ }
+}
+
+func TestAevo_FetchOI_UnknownAsset(t *testing.T) {
+ a := &Aevo{baseURL: "http://unused"}
+ _, err := a.FetchOI("SOL")
+ if err == nil {
+ t.Fatal("expected error for unsupported asset, got nil")
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_dydx.go b/harnesses/perp-liq-rate/cmd/script/source_dydx.go
new file mode 100644
index 00000000..f47362a5
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_dydx.go
@@ -0,0 +1,144 @@
+package main
+
+// source_dydx.go — dYdX v4 (indexer.dydx.trade).
+//
+// Liquidations: GET /v4/trades/perpetualMarket/{ticker}?limit=100, keep
+// type == "LIQUIDATED", paginate backwards with createdBeforeOrAt until the
+// page's oldest trade predates sinceMs.
+// OI: GET /v4/perpetualMarkets; openInterest (base units) * oraclePrice.
+
+import (
+ "fmt"
+ "net/url"
+ "time"
+)
+
+const (
+ dydxBaseURL = "https://indexer.dydx.trade/v4"
+ dydxPageSize = 100
+ dydxMaxPages = 30 // safety cap: 3000 trades per tick per market
+)
+
+var dydxTickers = map[string]string{
+ "ETH": "ETH-USD",
+ "BTC": "BTC-USD",
+ "SOL": "SOL-USD",
+}
+
+// Dydx implements Source. It is stateless and shared across assets.
+type Dydx struct {
+ baseURL string // defaults to dydxBaseURL
+}
+
+// NewDydx returns the dYdX source.
+func NewDydx() *Dydx { return &Dydx{baseURL: dydxBaseURL} }
+
+type dydxTrade struct {
+ ID string `json:"id"`
+ Size string `json:"size"`
+ Price string `json:"price"`
+ Type string `json:"type"`
+ CreatedAt string `json:"createdAt"` // ISO8601
+}
+
+type dydxTradesResp struct {
+ Trades []dydxTrade `json:"trades"`
+}
+
+// FetchLiquidationsSince pages the trade feed backwards until sinceMs.
+func (d *Dydx) FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error) {
+ ticker, ok := dydxTickers[asset]
+ if !ok {
+ return nil, fmt.Errorf("dydx: unsupported asset %q", asset)
+ }
+
+ var events []LiqEvent
+ createdBefore := ""
+
+ for page := 0; page < dydxMaxPages; page++ {
+ u := fmt.Sprintf("%s/trades/perpetualMarket/%s?limit=%d", d.baseURL, url.PathEscape(ticker), dydxPageSize)
+ if createdBefore != "" {
+ u += "&createdBeforeOrAt=" + url.QueryEscape(createdBefore)
+ }
+ var resp dydxTradesResp
+ if err := httpGetJSON(u, &resp); err != nil {
+ return nil, fmt.Errorf("dydx trades: %w", err)
+ }
+ if len(resp.Trades) == 0 {
+ break
+ }
+
+ oldestMs := int64(1<<62 - 1)
+ oldestStr := ""
+ for _, t := range resp.Trades {
+ ts, err := time.Parse(time.RFC3339Nano, t.CreatedAt)
+ if err != nil {
+ return nil, fmt.Errorf("dydx createdAt %q: %w", t.CreatedAt, err)
+ }
+ ms := ts.UnixMilli()
+ if ms < oldestMs {
+ oldestMs = ms
+ oldestStr = t.CreatedAt
+ }
+ if t.Type != "LIQUIDATED" || ms < sinceMs {
+ continue
+ }
+ sz, err := parseF(t.Size)
+ if err != nil {
+ return nil, fmt.Errorf("dydx size: %w", err)
+ }
+ px, err := parseF(t.Price)
+ if err != nil {
+ return nil, fmt.Errorf("dydx price: %w", err)
+ }
+ events = append(events, LiqEvent{
+ Key: t.ID,
+ NotionalUSD: sz * px,
+ TimestampMs: ms,
+ })
+ }
+
+ // Stop once the page reaches past our since bound or is short.
+ if oldestMs < sinceMs || len(resp.Trades) < dydxPageSize {
+ break
+ }
+ // createdBeforeOrAt is inclusive, so the boundary trade repeats on
+ // the next page; the SeenSet dedup absorbs that.
+ createdBefore = oldestStr
+ }
+ return events, nil
+}
+
+type dydxMarket struct {
+ OpenInterest string `json:"openInterest"`
+ OraclePrice string `json:"oraclePrice"`
+}
+
+// FetchOI returns openInterest * oraclePrice for the ticker.
+func (d *Dydx) FetchOI(asset string) (float64, error) {
+ ticker, ok := dydxTickers[asset]
+ if !ok {
+ return 0, fmt.Errorf("dydx: unsupported asset %q", asset)
+ }
+ var resp struct {
+ // VERIFY: the v4 indexer returns an object keyed by ticker under
+ // "markets" (the written spec said "array"; the live API is a map).
+ Markets map[string]dydxMarket `json:"markets"`
+ }
+ if err := httpGetJSON(d.baseURL+"/perpetualMarkets", &resp); err != nil {
+ return 0, fmt.Errorf("dydx perpetualMarkets: %w", err)
+ }
+ m, ok := resp.Markets[ticker]
+ if !ok {
+ return 0, fmt.Errorf("dydx: market %q not found", ticker)
+ }
+ oi, err := parseF(m.OpenInterest)
+ if err != nil {
+ return 0, fmt.Errorf("dydx openInterest: %w", err)
+ }
+ px, err := parseF(m.OraclePrice)
+ if err != nil {
+ return 0, fmt.Errorf("dydx oraclePrice: %w", err)
+ }
+ return oi * px, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_dydx_test.go b/harnesses/perp-liq-rate/cmd/script/source_dydx_test.go
new file mode 100644
index 00000000..1cc9d12e
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_dydx_test.go
@@ -0,0 +1,183 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func makeDydxTrade(id, tradeType string) map[string]any {
+ return map[string]any{
+ "id": id,
+ "size": "1.0",
+ "price": "3000.0",
+ "type": tradeType,
+ "createdAt": "2023-11-15T00:00:00.000Z",
+ }
+}
+
+func TestDydx_FetchLiquidationsSince_SinglePage(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !strings.Contains(r.URL.Path, "/trades/perpetualMarket/") {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ trades := []map[string]any{
+ makeDydxTrade("t1", "LIQUIDATED"),
+ makeDydxTrade("t2", "TAKER"),
+ makeDydxTrade("t3", "MAKER"),
+ }
+ resp := map[string]any{"trades": trades}
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ events, err := d.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 LIQUIDATED event, got %d", len(events))
+ }
+}
+
+func TestDydx_FetchLiquidationsSince_Pagination(t *testing.T) {
+ callCount := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ callCount++
+ var trades []map[string]any
+ if callCount == 1 {
+ // Full page: 100 TAKER trades + 1 LIQUIDATED at the start
+ for i := 0; i < 99; i++ {
+ trades = append(trades, map[string]any{
+ "id": fmt.Sprintf("taker-%d", i),
+ "size": "1.0",
+ "price": "3000.0",
+ "type": "TAKER",
+ "createdAt": "2023-11-15T01:00:00.000Z",
+ })
+ }
+ // One LIQUIDATED in the first page
+ trades = append(trades, map[string]any{
+ "id": "liq-1",
+ "size": "2.0",
+ "price": "3000.0",
+ "type": "LIQUIDATED",
+ "createdAt": "2023-11-15T01:00:01.000Z",
+ })
+ } else {
+ // Second page: 5 trades, 1 LIQUIDATED
+ for i := 0; i < 4; i++ {
+ trades = append(trades, map[string]any{
+ "id": fmt.Sprintf("taker2-%d", i),
+ "size": "1.0",
+ "price": "3000.0",
+ "type": "TAKER",
+ "createdAt": "2023-11-14T23:00:00.000Z",
+ })
+ }
+ trades = append(trades, map[string]any{
+ "id": "liq-2",
+ "size": "1.5",
+ "price": "3000.0",
+ "type": "LIQUIDATED",
+ "createdAt": "2023-11-14T23:00:01.000Z",
+ })
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"trades": trades})
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ events, err := d.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 LIQUIDATED events across pages, got %d", len(events))
+ }
+}
+
+func TestDydx_FetchLiquidationsSince_NonLiquidatedFiltered(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ trades := []map[string]any{
+ makeDydxTrade("t1", "TAKER"),
+ makeDydxTrade("t2", "MAKER"),
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"trades": trades})
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ events, err := d.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events, got %d", len(events))
+ }
+}
+
+func TestDydx_FetchLiquidationsSince_ServerError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ _, err := d.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error from 500, got nil")
+ }
+}
+
+func TestDydx_FetchOI_HappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/perpetualMarkets" {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ resp := map[string]any{
+ "markets": map[string]any{
+ "ETH-USD": map[string]any{
+ "openInterest": "10.0",
+ "oraclePrice": "3000.0",
+ },
+ },
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ oi, err := d.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // 10 * 3000 = 30000
+ if oi < 29999 || oi > 30001 {
+ t.Errorf("OI = %v, want 30000", oi)
+ }
+}
+
+func TestDydx_FetchOI_MarketNotFound(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "markets": map[string]any{
+ "BTC-USD": map[string]any{"openInterest": "1.0", "oraclePrice": "60000.0"},
+ },
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ d := &Dydx{baseURL: srv.URL}
+ _, err := d.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for missing market, got nil")
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_gains.go b/harnesses/perp-liq-rate/cmd/script/source_gains.go
new file mode 100644
index 00000000..fae27a98
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_gains.go
@@ -0,0 +1,488 @@
+package main
+
+// source_gains.go — Gains Network (gTrade) on Base mainnet.
+//
+// Liquidations: eth_getLogs on the diamond for TradeClosed events, keeping
+// only those whose cancelReason (last uint8 word of the event data) == 1.
+// Notional = collateralAmount/1e6 * leverage/1e3.
+// OI: GET backend-base.gains.trade/trading-variables; find pairIndex from
+// pairs[i].from == asset, then sum oiLongCollateral+oiShortCollateral from
+// the USDC collateral's pairOis[pairIndex] / 1e6.
+//
+// Because the only permitted external dependency is the Prometheus client,
+// a minimal Keccak-256 (legacy padding, as used for Ethereum event topics)
+// is implemented at the bottom of this file.
+
+import (
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "log"
+ "math/big"
+ "math/bits"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ gainsDiamond = "0x6cd5ac19a07518a8092eeffda4f1174c72704eeb"
+ gainsTradingVarsURL = "https://backend-base.gains.trade/trading-variables"
+
+ // ~24h of Base blocks at ~2s block time; also the hard cap on how far
+ // back we ever scan (older events fall outside the window anyway).
+ gainsInitialLookbackBlocks = 43200
+
+ // Conservative per-request block range for eth_getLogs on public RPC.
+ // VERIFY: https://mainnet.base.org getLogs range/result limits; raise if
+ // your provider allows larger ranges.
+ gainsMaxLogRangeBlocks = 5000
+
+ gainsBlockTimeMs = 2000 // Base ~2s blocks
+
+ // Sanity ceiling on a single decoded liquidation to guard against ABI
+ // word-offset mistakes producing nonsense notionals.
+ gainsMaxSingleNotionalUSD = 1e10
+)
+
+// gainsTradeClosedSig is hashed with Keccak-256 to obtain topic0.
+// VERIFY: exact TradeClosed signature against the deployed gains diamond ABI
+// on Base (field order inside the tuples in particular).
+const gainsTradeClosedSig = "TradeClosed(address,uint32,(address,uint32,bool,uint32,uint16,uint64,uint64),(uint32,uint64,uint64,bool,uint40,uint40),(uint256,int256,uint256,uint256,uint256,uint256,uint256,int256),(uint64,uint64,bool,bool),uint8)"
+
+var gainsTradeClosedTopic = func() string {
+ h := keccak256([]byte(gainsTradeClosedSig))
+ return "0x" + hex.EncodeToString(h[:])
+}()
+
+// gainsPairIndex maps assets to gTrade pair indices on the Base deployment.
+// Confirmed from trading-variables: pairs[0].from="BTC", pairs[1].from="ETH".
+var gainsPairIndex = map[string]uint64{
+ "BTC": 0,
+ "ETH": 1,
+}
+
+// Assumed word layout of the non-indexed event data. The data is a flat
+// sequence of 32-byte words because every tuple member is a static type.
+// Counting backwards from the end is robust to whether the two leading
+// scalar params (address, uint32) are indexed or inline:
+//
+// ... [tupleA: 7 words][tupleB: 6 words][tupleC: 8 words][tupleD: 4 words][cancelReason: 1 word]
+//
+// so tupleA starts at wordCount-26 and cancelReason is the last word.
+const gainsTailWords = 26 // 7 + 6 + 8 + 4 + 1
+
+// Offsets within tupleA (address,uint32,bool,uint32,uint16,uint64,uint64).
+const (
+ // VERIFY: position of pairIndex within the trade tuple in the gains ABI.
+ gainsTupleAWordPairIndex = 1
+ // VERIFY: position of leverage (1e3 fixed point) within the trade tuple.
+ gainsTupleAWordLeverage = 4
+ // VERIFY: position of collateralAmount (USDC, 6 decimals) within the
+ // trade tuple.
+ gainsTupleAWordCollateral = 5
+)
+
+const gainsCancelReasonLiquidation = 1
+
+// Gains implements Source via Base JSON-RPC log scanning.
+type Gains struct {
+ rpcURL string
+ tradingVarsURL string // defaults to gainsTradingVarsURL
+
+ mu sync.Mutex
+ lastBlock map[string]uint64 // per-asset processed high-water mark
+}
+
+// NewGains returns the gains source pointed at the given Base RPC URL.
+func NewGains(rpcURL string) *Gains {
+ return &Gains{
+ rpcURL: rpcURL,
+ tradingVarsURL: gainsTradingVarsURL,
+ lastBlock: make(map[string]uint64),
+ }
+}
+
+// --- JSON-RPC plumbing ---
+
+type rpcRequest struct {
+ Jsonrpc string `json:"jsonrpc"`
+ Method string `json:"method"`
+ Params []any `json:"params"`
+ ID int `json:"id"`
+}
+
+type rpcError struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+type rpcResponse struct {
+ Result json.RawMessage `json:"result"`
+ Error *rpcError `json:"error"`
+}
+
+func (g *Gains) rpcCall(method string, params []any, out any) error {
+ var resp rpcResponse
+ if err := httpPostJSON(g.rpcURL, rpcRequest{Jsonrpc: "2.0", Method: method, Params: params, ID: 1}, &resp); err != nil {
+ return fmt.Errorf("rpc %s: %w", method, err)
+ }
+ if resp.Error != nil {
+ return fmt.Errorf("rpc %s: code %d: %s", method, resp.Error.Code, resp.Error.Message)
+ }
+ if out != nil {
+ if err := json.Unmarshal(resp.Result, out); err != nil {
+ return fmt.Errorf("rpc %s decode: %w", method, err)
+ }
+ }
+ return nil
+}
+
+func (g *Gains) latestBlock() (uint64, error) {
+ var hexStr string
+ if err := g.rpcCall("eth_blockNumber", []any{}, &hexStr); err != nil {
+ return 0, err
+ }
+ return parseHexUint(hexStr)
+}
+
+type ethLog struct {
+ Address string `json:"address"`
+ Topics []string `json:"topics"`
+ Data string `json:"data"`
+ BlockNumber string `json:"blockNumber"`
+ TxHash string `json:"transactionHash"`
+ LogIndex string `json:"logIndex"`
+ Removed bool `json:"removed"`
+}
+
+// FetchLiquidationsSince scans TradeClosed logs from lastBlock+1 (first tick:
+// latest-43200) to latest and returns those decoded as liquidations of the
+// requested asset. sinceMs is unused: block cursoring replaces it here.
+func (g *Gains) FetchLiquidationsSince(asset string, _ int64) ([]LiqEvent, error) {
+ pairIdx, ok := gainsPairIndex[asset]
+ if !ok {
+ return nil, fmt.Errorf("gains: unsupported asset %q", asset)
+ }
+
+ latest, err := g.latestBlock()
+ if err != nil {
+ return nil, err
+ }
+
+ floor := uint64(1)
+ if latest > gainsInitialLookbackBlocks {
+ floor = latest - gainsInitialLookbackBlocks
+ }
+
+ g.mu.Lock()
+ from := g.lastBlock[asset] + 1
+ if g.lastBlock[asset] == 0 || from < floor {
+ // First tick, or we fell behind by more than the window: anything
+ // older than 24h would be pruned immediately, so clamp.
+ from = floor
+ }
+ g.mu.Unlock()
+
+ if from > latest {
+ return nil, nil
+ }
+
+ nowMs := time.Now().UnixMilli()
+ var events []LiqEvent
+
+ for start := from; start <= latest; start += gainsMaxLogRangeBlocks {
+ end := start + gainsMaxLogRangeBlocks - 1
+ if end > latest {
+ end = latest
+ }
+ filter := map[string]any{
+ "fromBlock": hexUint(start),
+ "toBlock": hexUint(end),
+ "address": gainsDiamond,
+ "topics": []any{gainsTradeClosedTopic},
+ }
+ var logs []ethLog
+ if err := g.rpcCall("eth_getLogs", []any{filter}, &logs); err != nil {
+ return nil, err
+ }
+ for _, lg := range logs {
+ if lg.Removed {
+ continue
+ }
+ ev, matched, decodeErr := decodeGainsTradeClosed(lg, pairIdx, latest, nowMs)
+ if decodeErr != nil {
+ // A single malformed log should not poison the whole tick;
+ // log and continue.
+ log.Printf("[gains/%s] skipping undecodable log %s:%s: %v", asset, lg.TxHash, lg.LogIndex, decodeErr)
+ continue
+ }
+ if matched {
+ events = append(events, ev)
+ }
+ }
+ }
+
+ // Advance the high-water mark only after the full range succeeded so a
+ // failed tick is retried from the same block next time.
+ g.mu.Lock()
+ if latest > g.lastBlock[asset] {
+ g.lastBlock[asset] = latest
+ }
+ g.mu.Unlock()
+
+ return events, nil
+}
+
+// decodeGainsTradeClosed decodes one TradeClosed log and reports whether it
+// is a liquidation of the wanted pair.
+func decodeGainsTradeClosed(lg ethLog, wantPair uint64, latest uint64, nowMs int64) (LiqEvent, bool, error) {
+ data, err := hexBytes(lg.Data)
+ if err != nil {
+ return LiqEvent{}, false, fmt.Errorf("data hex: %w", err)
+ }
+ if len(data) == 0 || len(data)%32 != 0 {
+ return LiqEvent{}, false, fmt.Errorf("data length %d not word-aligned", len(data))
+ }
+ words := len(data) / 32
+ if words < gainsTailWords {
+ return LiqEvent{}, false, fmt.Errorf("data has %d words, expected >= %d", words, gainsTailWords)
+ }
+ word := func(i int) *big.Int {
+ return new(big.Int).SetBytes(data[i*32 : (i+1)*32])
+ }
+
+ cancel := word(words - 1)
+ if !cancel.IsUint64() || cancel.Uint64() != gainsCancelReasonLiquidation {
+ return LiqEvent{}, false, nil // closed for another reason
+ }
+
+ base := words - gainsTailWords // first word of tupleA
+ pairWord := word(base + gainsTupleAWordPairIndex)
+ if !pairWord.IsUint64() {
+ return LiqEvent{}, false, fmt.Errorf("pairIndex word not uint64-representable")
+ }
+ if pairWord.Uint64() != wantPair {
+ return LiqEvent{}, false, nil // liquidation of a different pair
+ }
+
+ leverage := word(base + gainsTupleAWordLeverage)
+ collateral := word(base + gainsTupleAWordCollateral)
+
+ collateralF, _ := new(big.Float).SetInt(collateral).Float64()
+ leverageF, _ := new(big.Float).SetInt(leverage).Float64()
+ // collateral is USDC (6 decimals); leverage is 1e3 fixed point.
+ notional := collateralF / 1e6 * leverageF / 1e3
+ if notional <= 0 || notional > gainsMaxSingleNotionalUSD {
+ return LiqEvent{}, false, fmt.Errorf("implausible notional %.4f (check ABI word offsets)", notional)
+ }
+
+ blockNum, err := parseHexUint(lg.BlockNumber)
+ if err != nil {
+ return LiqEvent{}, false, fmt.Errorf("blockNumber: %w", err)
+ }
+ // Approximate the event time from block distance at ~2s per block; Base
+ // block times are stable enough for 24h windowing, and this avoids one
+ // eth_getBlockByNumber round trip per block.
+ var tsMs int64
+ if latest >= blockNum {
+ tsMs = nowMs - int64(latest-blockNum)*gainsBlockTimeMs
+ } else {
+ tsMs = nowMs
+ }
+
+ return LiqEvent{
+ Key: lg.TxHash + ":" + lg.LogIndex,
+ NotionalUSD: notional,
+ TimestampMs: tsMs,
+ }, true, nil
+}
+
+// gainsTV is the subset of the trading-variables response we care about.
+type gainsTV struct {
+ Pairs []struct {
+ From string `json:"from"`
+ } `json:"pairs"`
+ Collaterals []struct {
+ Symbol string `json:"symbol"`
+ PairOis []struct {
+ Collateral struct {
+ OILong string `json:"oiLongCollateral"`
+ OIShort string `json:"oiShortCollateral"`
+ } `json:"collateral"`
+ } `json:"pairOis"`
+ } `json:"collaterals"`
+}
+
+// FetchOI returns open interest in USD from the Gains trading-variables API.
+// It finds the pairIndex by matching pairs[i].from == asset, then sums
+// oiLongCollateral + oiShortCollateral from the USDC collateral / 1e6.
+func (g *Gains) FetchOI(asset string) (float64, error) {
+ var tv gainsTV
+ if err := httpGetJSON(g.tradingVarsURL, &tv); err != nil {
+ return 0, fmt.Errorf("gains trading-variables: %w", err)
+ }
+
+ pairIdx := -1
+ for i, p := range tv.Pairs {
+ if strings.EqualFold(p.From, asset) {
+ pairIdx = i
+ break
+ }
+ }
+ if pairIdx < 0 {
+ return 0, fmt.Errorf("gains: asset %q not found in pairs", asset)
+ }
+
+ var totalOI float64
+ for _, col := range tv.Collaterals {
+ if !strings.EqualFold(col.Symbol, "USDC") {
+ continue
+ }
+ if pairIdx >= len(col.PairOis) {
+ return 0, fmt.Errorf("gains: pairIdx %d out of range for collateral %s (len=%d)", pairIdx, col.Symbol, len(col.PairOis))
+ }
+ oiLong, err := parseF(col.PairOis[pairIdx].Collateral.OILong)
+ if err != nil {
+ return 0, fmt.Errorf("gains oiLongCollateral: %w", err)
+ }
+ oiShort, err := parseF(col.PairOis[pairIdx].Collateral.OIShort)
+ if err != nil {
+ return 0, fmt.Errorf("gains oiShortCollateral: %w", err)
+ }
+ totalOI += (oiLong + oiShort) / 1e6
+ }
+ if totalOI == 0 {
+ return 0, fmt.Errorf("gains: no USDC collateral OI found for %s", asset)
+ }
+ return totalOI, nil
+}
+
+// --- small hex helpers ---
+
+func hexUint(v uint64) string { return fmt.Sprintf("0x%x", v) }
+
+func parseHexUint(s string) (uint64, error) {
+ s = strings.TrimPrefix(strings.TrimSpace(s), "0x")
+ if s == "" {
+ return 0, fmt.Errorf("empty hex quantity")
+ }
+ v, err := strconv64(s)
+ if err != nil {
+ return 0, fmt.Errorf("parse hex %q: %w", s, err)
+ }
+ return v, nil
+}
+
+// strconv64 is a tiny wrapper kept separate for clarity at call sites.
+func strconv64(hexDigits string) (uint64, error) {
+ var v uint64
+ for _, c := range hexDigits {
+ var d uint64
+ switch {
+ case c >= '0' && c <= '9':
+ d = uint64(c - '0')
+ case c >= 'a' && c <= 'f':
+ d = uint64(c-'a') + 10
+ case c >= 'A' && c <= 'F':
+ d = uint64(c-'A') + 10
+ default:
+ return 0, fmt.Errorf("invalid hex digit %q", string(c))
+ }
+ if v > (^uint64(0))>>4 {
+ return 0, fmt.Errorf("hex quantity overflows uint64")
+ }
+ v = v<<4 | d
+ }
+ return v, nil
+}
+
+func hexBytes(s string) ([]byte, error) {
+ s = strings.TrimPrefix(strings.TrimSpace(s), "0x")
+ if len(s)%2 != 0 {
+ s = "0" + s
+ }
+ return hex.DecodeString(s)
+}
+
+// --- Keccak-256 (legacy padding, Ethereum-style) ---
+
+var keccakRC = [24]uint64{
+ 0x0000000000000001, 0x0000000000008082, 0x800000000000808A, 0x8000000080008000,
+ 0x000000000000808B, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009,
+ 0x000000000000008A, 0x0000000000000088, 0x0000000080008009, 0x000000008000000A,
+ 0x000000008000808B, 0x800000000000008B, 0x8000000000008089, 0x8000000000008003,
+ 0x8000000000008002, 0x8000000000000080, 0x000000000000800A, 0x800000008000000A,
+ 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008,
+}
+
+// keccakRot[x][y] are the rho rotation offsets for lane (x, y), lane index x+5y.
+var keccakRot = [5][5]int{
+ {0, 36, 3, 41, 18},
+ {1, 44, 10, 45, 2},
+ {62, 6, 43, 15, 61},
+ {28, 55, 25, 21, 56},
+ {27, 20, 39, 8, 14},
+}
+
+func keccakF(a *[25]uint64) {
+ var c, d [5]uint64
+ var b [25]uint64
+ for round := 0; round < 24; round++ {
+ // theta
+ for x := 0; x < 5; x++ {
+ c[x] = a[x] ^ a[x+5] ^ a[x+10] ^ a[x+15] ^ a[x+20]
+ }
+ for x := 0; x < 5; x++ {
+ d[x] = c[(x+4)%5] ^ bits.RotateLeft64(c[(x+1)%5], 1)
+ for y := 0; y < 5; y++ {
+ a[x+5*y] ^= d[x]
+ }
+ }
+ // rho + pi
+ for x := 0; x < 5; x++ {
+ for y := 0; y < 5; y++ {
+ b[y+5*((2*x+3*y)%5)] = bits.RotateLeft64(a[x+5*y], keccakRot[x][y])
+ }
+ }
+ // chi
+ for x := 0; x < 5; x++ {
+ for y := 0; y < 5; y++ {
+ a[x+5*y] = b[x+5*y] ^ ((^b[(x+1)%5+5*y]) & b[(x+2)%5+5*y])
+ }
+ }
+ // iota
+ a[0] ^= keccakRC[round]
+ }
+}
+
+// keccak256 computes the original Keccak-256 digest (0x01 padding, as used
+// by Ethereum for event topic hashing — not SHA3-256's 0x06 padding).
+func keccak256(data []byte) [32]byte {
+ const rate = 136 // bytes; 1088-bit rate for 256-bit output
+ var st [25]uint64
+
+ absorb := func(block []byte) {
+ for i := 0; i < rate/8; i++ {
+ st[i] ^= binary.LittleEndian.Uint64(block[i*8:])
+ }
+ keccakF(&st)
+ }
+
+ for len(data) >= rate {
+ absorb(data[:rate])
+ data = data[rate:]
+ }
+ var last [rate]byte
+ copy(last[:], data)
+ last[len(data)] ^= 0x01
+ last[rate-1] ^= 0x80
+ absorb(last[:])
+
+ var out [32]byte
+ for i := 0; i < 4; i++ {
+ binary.LittleEndian.PutUint64(out[i*8:], st[i])
+ }
+ return out
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_gains_test.go b/harnesses/perp-liq-rate/cmd/script/source_gains_test.go
new file mode 100644
index 00000000..8f3eac87
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_gains_test.go
@@ -0,0 +1,351 @@
+package main
+
+import (
+ "encoding/binary"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+// buildGainsData builds a 26-word (or 28-word if leadingWords>0) ABI-encoded
+// TradeClosed event data blob for testing decodeGainsTradeClosed.
+func buildGainsData(pairIdx uint32, leverage uint16, collateralUsdc uint64, cancelReason byte, leadingWords int) string {
+ total := 26 + leadingWords
+ data := make([]byte, total*32)
+ base := leadingWords * 32 // tupleA starts after any leading words
+
+ // word 1 of tupleA = pairIndex (uint32)
+ binary.BigEndian.PutUint32(data[base+1*32+28:base+1*32+32], pairIdx)
+ // word 4 of tupleA = leverage (uint16, 1e3 fixed point)
+ binary.BigEndian.PutUint16(data[base+4*32+30:base+4*32+32], leverage)
+ // word 5 of tupleA = collateral (USDC, 6 dp)
+ binary.BigEndian.PutUint64(data[base+5*32+24:base+5*32+32], collateralUsdc)
+ // last word = cancelReason
+ data[total*32-1] = cancelReason
+
+ return "0x" + hex.EncodeToString(data)
+}
+
+func makeEthLog(data, blockNum string) ethLog {
+ return ethLog{
+ Data: data,
+ BlockNumber: blockNum,
+ TxHash: "0xtxhash",
+ LogIndex: "0x0",
+ }
+}
+
+func TestDecodeGainsTradeClosed_Liquidation(t *testing.T) {
+ // leverage=10000 (10x, 1e3 precision), collateral=1000000000 (1000 USDC at 6dp)
+ // notional = 1000000000/1e6 * 10000/1e3 = 1000 * 10 = 10000
+ data := buildGainsData(0, 10000, 1000000000, 1, 0)
+ lg := makeEthLog(data, "0x100")
+ ev, matched, err := decodeGainsTradeClosed(lg, 0, 0x100, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !matched {
+ t.Fatal("expected matched=true")
+ }
+ if ev.NotionalUSD < 9999 || ev.NotionalUSD > 10001 {
+ t.Errorf("notional = %v, want ~10000", ev.NotionalUSD)
+ }
+}
+
+func TestDecodeGainsTradeClosed_NotLiquidation(t *testing.T) {
+ data := buildGainsData(0, 10000, 1000000000, 0, 0) // cancelReason=0
+ lg := makeEthLog(data, "0x100")
+ _, matched, err := decodeGainsTradeClosed(lg, 0, 0x100, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if matched {
+ t.Fatal("expected matched=false for cancelReason=0")
+ }
+}
+
+func TestDecodeGainsTradeClosed_WrongPair(t *testing.T) {
+ // pairIndex=1, wantPair=0
+ data := buildGainsData(1, 10000, 1000000000, 1, 0)
+ lg := makeEthLog(data, "0x100")
+ _, matched, err := decodeGainsTradeClosed(lg, 0, 0x100, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if matched {
+ t.Fatal("expected matched=false for wrong pair")
+ }
+}
+
+func TestDecodeGainsTradeClosed_ShortData(t *testing.T) {
+ // Only 10 words — less than gainsTailWords (26)
+ data := "0x" + hex.EncodeToString(make([]byte, 10*32))
+ lg := makeEthLog(data, "0x100")
+ _, _, err := decodeGainsTradeClosed(lg, 0, 0x100, 0)
+ if err == nil {
+ t.Fatal("expected error for short data, got nil")
+ }
+}
+
+func TestDecodeGainsTradeClosed_28Words(t *testing.T) {
+ // 28-word data (leading 2 words for address+uint32 inline, not indexed)
+ data := buildGainsData(0, 10000, 1000000000, 1, 2)
+ lg := makeEthLog(data, "0x100")
+ ev, matched, err := decodeGainsTradeClosed(lg, 0, 0x100, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !matched {
+ t.Fatal("expected matched=true for 28-word data")
+ }
+ if ev.NotionalUSD < 9999 || ev.NotionalUSD > 10001 {
+ t.Errorf("notional = %v, want ~10000", ev.NotionalUSD)
+ }
+}
+
+// mockRPCServer returns a simple JSON-RPC server handling eth_blockNumber and eth_getLogs.
+// All eth_getLogs calls return the provided logs slice.
+func mockRPCServer(t *testing.T, blockHex string, logs []map[string]any) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req rpcRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ var result any
+ switch req.Method {
+ case "eth_blockNumber":
+ result = blockHex
+ case "eth_getLogs":
+ result = logs
+ default:
+ result = nil
+ }
+ resp := map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result}
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+}
+
+func TestGains_FetchLiquidationsSince_HappyPath(t *testing.T) {
+ // Build 2 valid liquidation log data blobs for ETH (pair 1 on Base deployment)
+ data1 := buildGainsData(1, 10000, 1000000000, 1, 0)
+ data2 := buildGainsData(1, 5000, 2000000000, 1, 0)
+
+ logs := []map[string]any{
+ {
+ "address": gainsDiamond,
+ "topics": []string{gainsTradeClosedTopic},
+ "data": data1,
+ "blockNumber": "0x64",
+ "transactionHash": "0xtx1",
+ "logIndex": "0x0",
+ "removed": false,
+ },
+ {
+ "address": gainsDiamond,
+ "topics": []string{gainsTradeClosedTopic},
+ "data": data2,
+ "blockNumber": "0x64",
+ "transactionHash": "0xtx2",
+ "logIndex": "0x1",
+ "removed": false,
+ },
+ }
+
+ // Use block 100 — small enough that the entire range [1..100] fits in one
+ // getLogs call (< gainsMaxLogRangeBlocks=5000), so the server returns the
+ // 2 logs exactly once.
+ srv := mockRPCServer(t, "0x64", logs) // block 100
+ defer srv.Close()
+
+ g := NewGains(srv.URL)
+ events, err := g.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 events, got %d", len(events))
+ }
+}
+
+func TestGains_FetchLiquidationsSince_RPCError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ g := NewGains(srv.URL)
+ _, err := g.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error from RPC failure, got nil")
+ }
+}
+
+func TestGains_FetchLiquidationsSince_LastBlockAdvances(t *testing.T) {
+ srv := mockRPCServer(t, "0x100", []map[string]any{})
+ defer srv.Close()
+
+ g := NewGains(srv.URL)
+ _, err := g.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ g.mu.Lock()
+ lb := g.lastBlock["ETH"]
+ g.mu.Unlock()
+ if lb == 0 {
+ t.Error("lastBlock should advance after successful fetch")
+ }
+}
+
+func buildGainsTVResponse(pairs []map[string]any, collaterals []map[string]any) map[string]any {
+ return map[string]any{"pairs": pairs, "collaterals": collaterals}
+}
+
+func gainsUSDCCollateral(pairOis []map[string]any) map[string]any {
+ return map[string]any{"symbol": "USDC", "pairOis": pairOis}
+}
+
+func gainsPairOI(oiLong, oiShort string) map[string]any {
+ return map[string]any{"collateral": map[string]any{
+ "oiLongCollateral": oiLong,
+ "oiShortCollateral": oiShort,
+ }}
+}
+
+func TestGains_FetchOI_HappyPath(t *testing.T) {
+ // pairs: BTC=0, ETH=1 (matches live API ordering)
+ // ETH USDC OI: long=28438439935, short=27484847710 → (55923287645)/1e6 ≈ 55923.29 USD
+ pairs := []map[string]any{{"from": "BTC"}, {"from": "ETH"}}
+ pairOis := []map[string]any{
+ gainsPairOI("74229636084", "86846986430"), // BTC
+ gainsPairOI("28438439935", "27484847710"), // ETH
+ }
+ collaterals := []map[string]any{gainsUSDCCollateral(pairOis)}
+ body := buildGainsTVResponse(pairs, collaterals)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(body)
+ }))
+ defer srv.Close()
+
+ g := &Gains{rpcURL: "unused", tradingVarsURL: srv.URL, lastBlock: make(map[string]uint64)}
+ oi, err := g.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // (28438439935 + 27484847710) / 1e6 = 55923.287645
+ if oi < 55923 || oi > 55924 {
+ t.Errorf("OI = %v, want ~55923", oi)
+ }
+}
+
+func TestGains_FetchOI_AssetCaseInsensitive(t *testing.T) {
+ pairs := []map[string]any{{"from": "eth"}}
+ pairOis := []map[string]any{gainsPairOI("10000000", "5000000")} // 15 USD
+ collaterals := []map[string]any{gainsUSDCCollateral(pairOis)}
+ body := buildGainsTVResponse(pairs, collaterals)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(body)
+ }))
+ defer srv.Close()
+
+ g := &Gains{rpcURL: "unused", tradingVarsURL: srv.URL, lastBlock: make(map[string]uint64)}
+ oi, err := g.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // (10000000 + 5000000) / 1e6 = 15
+ if oi < 14.9 || oi > 15.1 {
+ t.Errorf("OI = %v, want ~15", oi)
+ }
+}
+
+func TestGains_FetchOI_UnknownAsset(t *testing.T) {
+ pairs := []map[string]any{{"from": "BTC"}}
+ pairOis := []map[string]any{gainsPairOI("1000000", "1000000")}
+ collaterals := []map[string]any{gainsUSDCCollateral(pairOis)}
+ body := buildGainsTVResponse(pairs, collaterals)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(body)
+ }))
+ defer srv.Close()
+
+ g := &Gains{rpcURL: "unused", tradingVarsURL: srv.URL, lastBlock: make(map[string]uint64)}
+ _, err := g.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for unknown asset, got nil")
+ }
+}
+
+func TestGains_FetchOI_SkipsNonUSDCCollateral(t *testing.T) {
+ pairs := []map[string]any{{"from": "ETH"}}
+ // Only a non-USDC collateral — should return error (no USDC OI found)
+ pairOis := []map[string]any{gainsPairOI("5000000", "3000000")}
+ collaterals := []map[string]any{{"symbol": "BtcUSD", "pairOis": pairOis}}
+ body := buildGainsTVResponse(pairs, collaterals)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(body)
+ }))
+ defer srv.Close()
+
+ g := &Gains{rpcURL: "unused", tradingVarsURL: srv.URL, lastBlock: make(map[string]uint64)}
+ _, err := g.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error when no USDC collateral found")
+ }
+}
+
+func TestGains_FetchLiquidationsSince_LastBlockNoAdvanceOnError(t *testing.T) {
+ callCount := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req rpcRequest
+ _ = json.NewDecoder(r.Body).Decode(&req)
+ callCount++
+ if req.Method == "eth_blockNumber" {
+ resp := map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": "0x100"}
+ _ = json.NewEncoder(w).Encode(resp)
+ return
+ }
+ // eth_getLogs fails
+ resp := map[string]any{
+ "jsonrpc": "2.0",
+ "id": req.ID,
+ "error": map[string]any{"code": -32000, "message": "getLogs error"},
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ g := NewGains(srv.URL)
+ _, err := g.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error from getLogs failure")
+ }
+ g.mu.Lock()
+ lb := g.lastBlock["ETH"]
+ g.mu.Unlock()
+ if lb != 0 {
+ t.Errorf("lastBlock should not advance on error, got %d", lb)
+ }
+}
+
+func TestGains_FetchLiquidationsSince_UnknownAsset(t *testing.T) {
+ g := NewGains("http://unused")
+ _, err := g.FetchLiquidationsSince("DOGE", 0)
+ if err == nil {
+ t.Fatal("expected error for unsupported asset")
+ }
+}
+
+func init() {
+ // Suppress log output in tests
+ _ = fmt.Sprintf // keep fmt imported
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_gmx.go b/harnesses/perp-liq-rate/cmd/script/source_gmx.go
new file mode 100644
index 00000000..66b5f78f
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_gmx.go
@@ -0,0 +1,125 @@
+package main
+
+// source_gmx.go — GMX v2 on Arbitrum.
+//
+// OI: GET arbitrum-api.gmxinfra.io/markets/info — sums openInterestLong +
+// openInterestShort across all isListed markets whose name matches the asset
+// (e.g. "ETH/USD [ETH-USDC]" + "ETH/USD [ETH-ETH]" for ETH). Values are
+// 30-decimal USD strings; divided by 1e30 to get USD. ✓
+// Liquidations: the TheGraph gmx-v2 subgraph is defunct; no alternative
+// public source found. FetchLiquidationsSince returns empty; liq_rate = 0.
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ gmxMarketsInfoURL = "https://arbitrum-api.gmxinfra.io/markets/info"
+ gmxMarketsTTL = 4 * time.Minute
+)
+
+// gmxTrackedAssets lists the assets supported by this source.
+var gmxTrackedAssets = map[string]bool{"ETH": true, "BTC": true}
+
+// GMX implements Source with a small cached /markets/info snapshot.
+type GMX struct {
+ marketsURL string // defaults to gmxMarketsInfoURL
+
+ mu sync.Mutex
+ markets []gmxMarket
+ marketsAt time.Time
+}
+
+// NewGMX returns the GMX source.
+func NewGMX() *GMX {
+ return &GMX{marketsURL: gmxMarketsInfoURL}
+}
+
+type gmxMarket struct {
+ Name string `json:"name"`
+ MarketToken string `json:"marketToken"`
+ IsListed bool `json:"isListed"`
+ OpenInterestLong string `json:"openInterestLong"` // 30-decimal USD
+ OpenInterestShort string `json:"openInterestShort"` // 30-decimal USD
+}
+
+func (g *GMX) fetchMarkets() ([]gmxMarket, error) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.markets != nil && time.Since(g.marketsAt) < gmxMarketsTTL {
+ return g.markets, nil
+ }
+ var resp struct {
+ Markets []gmxMarket `json:"markets"`
+ }
+ if err := httpGetJSON(g.marketsURL, &resp); err != nil {
+ return nil, fmt.Errorf("gmx markets: %w", err)
+ }
+ if len(resp.Markets) == 0 {
+ return nil, fmt.Errorf("gmx markets: empty market list")
+ }
+ g.markets = resp.Markets
+ g.marketsAt = time.Now()
+ return g.markets, nil
+}
+
+// gmxAssetMatches reports whether the asset appears as the leading token of
+// the market name (before the first "/" or space), preventing "WETH/..." from
+// matching "ETH" while still matching "ETH/USD [WETH-USDC]" and similar.
+func gmxAssetMatches(marketName, asset string) bool {
+ upper := strings.ToUpper(strings.TrimSpace(marketName))
+ a := strings.ToUpper(asset)
+ return upper == a ||
+ strings.HasPrefix(upper, a+"/") ||
+ strings.HasPrefix(upper, a+" ") ||
+ strings.HasPrefix(upper, a+"-")
+}
+
+// FetchLiquidationsSince returns empty — no public GMX V2 liquidation source
+// is available (TheGraph subgraph defunct, no REST alternative found).
+func (g *GMX) FetchLiquidationsSince(asset string, _ int64) ([]LiqEvent, error) {
+ if !gmxTrackedAssets[asset] {
+ return nil, fmt.Errorf("gmx: unsupported asset %q", asset)
+ }
+ return nil, nil
+}
+
+// FetchOI sums openInterestLong + openInterestShort across all listed markets
+// that match the asset, converting from 30-decimal fixed-point USD to float64.
+func (g *GMX) FetchOI(asset string) (float64, error) {
+ if !gmxTrackedAssets[asset] {
+ return 0, fmt.Errorf("gmx: unsupported asset %q", asset)
+ }
+ markets, err := g.fetchMarkets()
+ if err != nil {
+ return 0, err
+ }
+
+ var totalOI float64
+ matched := false
+ for _, m := range markets {
+ if !m.IsListed || !gmxAssetMatches(m.Name, asset) {
+ continue
+ }
+ matched = true
+ if m.OpenInterestLong != "" {
+ v, err := parseScaled(m.OpenInterestLong, 30)
+ if err == nil {
+ totalOI += v
+ }
+ }
+ if m.OpenInterestShort != "" {
+ v, err := parseScaled(m.OpenInterestShort, 30)
+ if err == nil {
+ totalOI += v
+ }
+ }
+ }
+ if !matched {
+ return 0, fmt.Errorf("gmx: no listed markets found for %q", asset)
+ }
+ return totalOI, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_gmx_test.go b/harnesses/perp-liq-rate/cmd/script/source_gmx_test.go
new file mode 100644
index 00000000..c832dd17
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_gmx_test.go
@@ -0,0 +1,170 @@
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestGmxAssetMatches(t *testing.T) {
+ cases := []struct {
+ name string
+ asset string
+ want bool
+ }{
+ {"ETH/USD [WETH-USDC]", "ETH", true},
+ {"BTC/USD", "BTC", true},
+ {"WETH/USD", "ETH", false},
+ {"SOL/USD", "ETH", false},
+ {"ETH-PERP", "ETH", true},
+ {"ETH", "ETH", true},
+ {"WETH", "ETH", false},
+ {"ETH USD", "ETH", true},
+ }
+ for _, c := range cases {
+ got := gmxAssetMatches(c.name, c.asset)
+ if got != c.want {
+ t.Errorf("gmxAssetMatches(%q, %q) = %v, want %v", c.name, c.asset, got, c.want)
+ }
+ }
+}
+
+func buildGMXInfoServer(t *testing.T, markets []map[string]any) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(map[string]any{"markets": markets})
+ }))
+}
+
+func gmxMarketEntry(name, token string, isListed bool, oiLong, oiShort string) map[string]any {
+ return map[string]any{
+ "name": name,
+ "marketToken": token,
+ "isListed": isListed,
+ "openInterestLong": oiLong,
+ "openInterestShort": oiShort,
+ }
+}
+
+// 5000 USD in 30-decimal: 5000 * 1e30
+const gmxOI5000 = "5000000000000000000000000000000000"
+
+// 3000 USD in 30-decimal
+const gmxOI3000 = "3000000000000000000000000000000000"
+
+func TestGMX_FetchOI_HappyPath(t *testing.T) {
+ markets := []map[string]any{
+ gmxMarketEntry("ETH/USD [ETH-USDC]", "0xETH1", true, gmxOI5000, gmxOI3000),
+ gmxMarketEntry("ETH/USD [ETH-ETH]", "0xETH2", true, gmxOI5000, gmxOI5000),
+ gmxMarketEntry("BTC/USD [WBTC-USDC]", "0xBTC1", true, gmxOI3000, gmxOI3000),
+ }
+ srv := buildGMXInfoServer(t, markets)
+ defer srv.Close()
+
+ g := &GMX{marketsURL: srv.URL}
+ oi, err := g.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // ETH: (5000+3000) + (5000+5000) = 18000
+ if oi < 17999 || oi > 18001 {
+ t.Errorf("OI = %v, want ~18000", oi)
+ }
+}
+
+func TestGMX_FetchOI_SkipsUnlisted(t *testing.T) {
+ markets := []map[string]any{
+ gmxMarketEntry("ETH/USD [ETH-USDC]", "0xETH1", true, gmxOI5000, gmxOI3000),
+ gmxMarketEntry("ETH/USD [ETH-synth]", "0xETH2", false, gmxOI5000, gmxOI5000), // unlisted
+ }
+ srv := buildGMXInfoServer(t, markets)
+ defer srv.Close()
+
+ g := &GMX{marketsURL: srv.URL}
+ oi, err := g.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // Only the listed market: 5000 + 3000 = 8000
+ if oi < 7999 || oi > 8001 {
+ t.Errorf("OI = %v, want ~8000 (unlisted market excluded)", oi)
+ }
+}
+
+func TestGMX_FetchOI_NoMarketsForAsset(t *testing.T) {
+ markets := []map[string]any{
+ gmxMarketEntry("BTC/USD [WBTC-USDC]", "0xBTC1", true, gmxOI5000, gmxOI3000),
+ }
+ srv := buildGMXInfoServer(t, markets)
+ defer srv.Close()
+
+ g := &GMX{marketsURL: srv.URL}
+ _, err := g.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for asset with no listed markets, got nil")
+ }
+}
+
+func TestGMX_FetchOI_MarketsFail(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ g := &GMX{marketsURL: srv.URL}
+ _, err := g.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error when markets endpoint fails, got nil")
+ }
+}
+
+func TestGMX_FetchOI_UnknownAsset(t *testing.T) {
+ g := &GMX{marketsURL: "http://unused"}
+ _, err := g.FetchOI("SOL")
+ if err == nil {
+ t.Fatal("expected error for unsupported asset, got nil")
+ }
+}
+
+func TestGMX_FetchLiquidationsSince_ReturnsEmpty(t *testing.T) {
+ g := &GMX{marketsURL: "http://unused"}
+ events, err := g.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events (no liquidation source), got %d", len(events))
+ }
+}
+
+func TestGMX_FetchLiquidationsSince_UnknownAsset(t *testing.T) {
+ g := &GMX{marketsURL: "http://unused"}
+ _, err := g.FetchLiquidationsSince("DOGE", 0)
+ if err == nil {
+ t.Fatal("expected error for unsupported asset, got nil")
+ }
+}
+
+func TestGMX_MarketsCache(t *testing.T) {
+ callCount := 0
+ markets := []map[string]any{
+ gmxMarketEntry("ETH/USD [ETH-USDC]", "0xETH1", true, gmxOI5000, gmxOI3000),
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ callCount++
+ _ = json.NewEncoder(w).Encode(map[string]any{"markets": markets})
+ }))
+ defer srv.Close()
+
+ g := &GMX{marketsURL: srv.URL}
+ if _, err := g.FetchOI("ETH"); err != nil {
+ t.Fatalf("first FetchOI error: %v", err)
+ }
+ if _, err := g.FetchOI("ETH"); err != nil {
+ t.Fatalf("second FetchOI error: %v", err)
+ }
+ if callCount != 1 {
+ t.Errorf("markets endpoint called %d times, want 1 (cache hit on second call)", callCount)
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_hyperliquid.go b/harnesses/perp-liq-rate/cmd/script/source_hyperliquid.go
new file mode 100644
index 00000000..2aebdf22
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_hyperliquid.go
@@ -0,0 +1,286 @@
+package main
+
+// source_hyperliquid.go — Hyperliquid perps.
+//
+// Liquidations: 0xArchive REST API when OXARCHIVE_API_KEY is set (full
+// coverage — all liquidation types including market-order fills). Fallback
+// without a key: userFillsByTime on the HLP liquidator vault (backstop only,
+// minority of volume).
+// OI: POST /info {"type":"metaAndAssetCtxs"}; openInterest * midPx. ✓
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ hyperliquidInfoURL = "https://api.hyperliquid.xyz/info"
+ hlpLiquidatorAddr = "0x2e3d94f0562703b25c83308a05046ddaf9a8dd14"
+ oxArchiveBaseURL = "https://api.0xarchive.io"
+ oxaMaxPages = 20
+ oxaPageLimit = 1000
+)
+
+// Hyperliquid implements Source. It is stateless and shared across assets.
+type Hyperliquid struct {
+ infoURL string // defaults to hyperliquidInfoURL
+ archiveBaseURL string // 0xArchive API base, defaults to oxArchiveBaseURL
+ archiveAPIKey string // from env OXARCHIVE_API_KEY
+}
+
+// NewHyperliquid returns the Hyperliquid source.
+func NewHyperliquid() *Hyperliquid {
+ return &Hyperliquid{
+ infoURL: hyperliquidInfoURL,
+ archiveBaseURL: oxArchiveBaseURL,
+ archiveAPIKey: os.Getenv("OXARCHIVE_API_KEY"),
+ }
+}
+
+var hyperliquidCoins = map[string]string{
+ "ETH": "ETH",
+ "BTC": "BTC",
+ "SOL": "SOL",
+}
+
+type hlFill struct {
+ Coin string `json:"coin"`
+ Px string `json:"px"`
+ Sz string `json:"sz"`
+ Time int64 `json:"time"` // unix ms
+ Hash string `json:"hash"`
+ Tid json.Number `json:"tid"`
+ Liquidation json.RawMessage `json:"liquidation"`
+}
+
+type oxaLiquidation struct {
+ Coin string `json:"coin"`
+ Timestamp string `json:"timestamp"` // ISO 8601 with ms
+ Price string `json:"price"`
+ Size string `json:"size"`
+ TradeID int64 `json:"trade_id"`
+}
+
+type oxaResponse struct {
+ Data []oxaLiquidation `json:"data"`
+ Meta struct {
+ NextCursor *string `json:"next_cursor"`
+ } `json:"meta"`
+}
+
+// httpGetJSONKey performs a GET with an X-API-Key header.
+func httpGetJSONKey(u, apiKey string, out any) error {
+ req, err := http.NewRequest(http.MethodGet, u, nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("X-API-Key", apiKey)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return &httpStatusError{Code: resp.StatusCode}
+ }
+ return json.NewDecoder(resp.Body).Decode(out)
+}
+
+// fetchOxaLiquidations pages 0xArchive for all HL liquidation events.
+func (h *Hyperliquid) fetchOxaLiquidations(coin string, sinceMs int64) ([]LiqEvent, error) {
+ startISO := time.UnixMilli(sinceMs).UTC().Format(time.RFC3339)
+ endISO := time.Now().UTC().Format(time.RFC3339)
+
+ var events []LiqEvent
+ cursor := ""
+
+ for page := 0; page < oxaMaxPages; page++ {
+ u := fmt.Sprintf("%s/v1/hyperliquid/liquidations/%s?start_time=%s&end_time=%s&limit=%d",
+ h.archiveBaseURL,
+ url.PathEscape(coin),
+ url.QueryEscape(startISO),
+ url.QueryEscape(endISO),
+ oxaPageLimit,
+ )
+ if cursor != "" {
+ u += "&cursor=" + url.QueryEscape(cursor)
+ }
+
+ var resp oxaResponse
+ if err := httpGetJSONKey(u, h.archiveAPIKey, &resp); err != nil {
+ return nil, fmt.Errorf("hyperliquid 0xarchive: %w", err)
+ }
+
+ for _, liq := range resp.Data {
+ px, err := parseF(liq.Price)
+ if err != nil {
+ return nil, fmt.Errorf("hyperliquid oxa px: %w", err)
+ }
+ sz, err := parseF(liq.Size)
+ if err != nil {
+ return nil, fmt.Errorf("hyperliquid oxa sz: %w", err)
+ }
+ ts, err := time.Parse(time.RFC3339Nano, liq.Timestamp)
+ if err != nil {
+ ts, err = time.Parse(time.RFC3339, liq.Timestamp)
+ if err != nil {
+ return nil, fmt.Errorf("hyperliquid oxa timestamp %q: %w", liq.Timestamp, err)
+ }
+ }
+ events = append(events, LiqEvent{
+ Key: "oxa:" + strconv.FormatInt(liq.TradeID, 10),
+ NotionalUSD: px * sz,
+ TimestampMs: ts.UnixMilli(),
+ })
+ }
+
+ if resp.Meta.NextCursor == nil || *resp.Meta.NextCursor == "" {
+ break
+ }
+ cursor = *resp.Meta.NextCursor
+ }
+ return events, nil
+}
+
+// FetchLiquidationsSince returns liquidation events newer than sinceMs.
+// Uses 0xArchive when OXARCHIVE_API_KEY is set; otherwise falls back to the
+// HLP liquidator vault (backstop liquidations only).
+func (h *Hyperliquid) FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error) {
+ coin, ok := hyperliquidCoins[asset]
+ if !ok {
+ return nil, fmt.Errorf("hyperliquid: unsupported asset %q", asset)
+ }
+
+ if h.archiveAPIKey != "" {
+ return h.fetchOxaLiquidations(coin, sinceMs)
+ }
+
+ // Backstop-only fallback via HLP liquidator vault.
+ payload := map[string]any{
+ "type": "userFillsByTime",
+ "user": hlpLiquidatorAddr,
+ "startTime": sinceMs,
+ }
+ var fills []hlFill
+ if err := httpPostJSON(h.infoURL, payload, &fills); err != nil {
+ return nil, fmt.Errorf("hyperliquid userFillsByTime: %w", err)
+ }
+
+ events := make([]LiqEvent, 0, 8)
+ for _, f := range fills {
+ if !jsonNonNull(f.Liquidation) {
+ continue
+ }
+ if f.Coin != coin {
+ continue
+ }
+ if f.Time < sinceMs {
+ continue
+ }
+ px, err := parseF(f.Px)
+ if err != nil {
+ return nil, fmt.Errorf("hyperliquid px: %w", err)
+ }
+ sz, err := parseF(f.Sz)
+ if err != nil {
+ return nil, fmt.Errorf("hyperliquid sz: %w", err)
+ }
+ key := f.Hash
+ if key == "" || isZeroHash(key) {
+ if f.Tid.String() != "" {
+ key = "tid:" + f.Tid.String()
+ } else {
+ key = fmt.Sprintf("hl:%s:%d:%s:%s", coin, f.Time, f.Px, f.Sz)
+ }
+ }
+ events = append(events, LiqEvent{
+ Key: key,
+ NotionalUSD: px * sz,
+ TimestampMs: f.Time,
+ })
+ }
+ return events, nil
+}
+
+// FetchOI returns open interest in USD: openInterest (base units) * midPx.
+func (h *Hyperliquid) FetchOI(asset string) (float64, error) {
+ coin, ok := hyperliquidCoins[asset]
+ if !ok {
+ return 0, fmt.Errorf("hyperliquid: unsupported asset %q", asset)
+ }
+
+ payload := map[string]any{"type": "metaAndAssetCtxs"}
+ var raw []json.RawMessage
+ if err := httpPostJSON(h.infoURL, payload, &raw); err != nil {
+ return 0, fmt.Errorf("hyperliquid metaAndAssetCtxs: %w", err)
+ }
+ if len(raw) < 2 {
+ return 0, fmt.Errorf("hyperliquid metaAndAssetCtxs: expected 2-element array, got %d", len(raw))
+ }
+
+ var meta struct {
+ Universe []struct {
+ Name string `json:"name"`
+ } `json:"universe"`
+ }
+ if err := json.Unmarshal(raw[0], &meta); err != nil {
+ return 0, fmt.Errorf("hyperliquid universe decode: %w", err)
+ }
+ var ctxs []struct {
+ OpenInterest string `json:"openInterest"`
+ MidPx string `json:"midPx"`
+ MarkPx string `json:"markPx"`
+ }
+ if err := json.Unmarshal(raw[1], &ctxs); err != nil {
+ return 0, fmt.Errorf("hyperliquid assetCtxs decode: %w", err)
+ }
+
+ for i, u := range meta.Universe {
+ if u.Name != coin {
+ continue
+ }
+ if i >= len(ctxs) {
+ return 0, fmt.Errorf("hyperliquid: assetCtxs index %d out of range (%d)", i, len(ctxs))
+ }
+ oi, err := parseF(ctxs[i].OpenInterest)
+ if err != nil {
+ return 0, fmt.Errorf("hyperliquid openInterest: %w", err)
+ }
+ pxStr := ctxs[i].MidPx
+ if strings.TrimSpace(pxStr) == "" {
+ pxStr = ctxs[i].MarkPx
+ }
+ px, err := parseF(pxStr)
+ if err != nil {
+ return 0, fmt.Errorf("hyperliquid midPx: %w", err)
+ }
+ return oi * px, nil
+ }
+ return 0, fmt.Errorf("hyperliquid: coin %q not found in universe", coin)
+}
+
+// jsonNonNull reports whether a raw JSON field was present and not null.
+func jsonNonNull(raw json.RawMessage) bool {
+ s := strings.TrimSpace(string(raw))
+ return s != "" && s != "null"
+}
+
+// isZeroHash reports whether s is a 0x-prefixed all-zero hash.
+func isZeroHash(s string) bool {
+ s = strings.TrimPrefix(strings.ToLower(s), "0x")
+ if s == "" {
+ return false
+ }
+ for _, c := range s {
+ if c != '0' {
+ return false
+ }
+ }
+ return true
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_hyperliquid_test.go b/harnesses/perp-liq-rate/cmd/script/source_hyperliquid_test.go
new file mode 100644
index 00000000..df25c4b4
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_hyperliquid_test.go
@@ -0,0 +1,299 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func hlFillJSON(coin, px, sz string, ts int64, hash, tid string, liq any) string {
+ liqStr := "null"
+ if liq != nil {
+ b, _ := json.Marshal(liq)
+ liqStr = string(b)
+ }
+ return fmt.Sprintf(`{"coin":%q,"px":%q,"sz":%q,"time":%d,"hash":%q,"tid":%q,"liquidation":%s}`,
+ coin, px, sz, ts, hash, tid, liqStr)
+}
+
+func TestHyperliquid_FetchLiquidationsSince_HappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&req)
+ if req["type"] == "userFillsByTime" {
+ fmt.Fprintf(w, `[%s,%s,%s]`,
+ hlFillJSON("ETH", "3000.0", "1.5", 1700000000000, "0xabc123", "1", map[string]any{"liq": true}),
+ hlFillJSON("ETH", "3000.0", "2.0", 1700000001000, "0xdef456", "2", nil),
+ hlFillJSON("ETH", "3100.0", "0.5", 1700000002000, "0xghi789", "3", map[string]any{"liq": true}),
+ )
+ }
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 events, got %d", len(events))
+ }
+ // 3000 * 1.5 = 4500
+ if events[0].NotionalUSD != 4500.0 {
+ t.Errorf("event[0] notional = %v, want 4500", events[0].NotionalUSD)
+ }
+ // 3100 * 0.5 = 1550
+ if events[1].NotionalUSD != 1550.0 {
+ t.Errorf("event[1] notional = %v, want 1550", events[1].NotionalUSD)
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_CoinFilter(t *testing.T) {
+ // Vault returns fills for multiple coins; only ETH should be kept.
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, `[%s,%s]`,
+ hlFillJSON("ETH", "3000.0", "1.0", 1700000000000, "0xaaa", "1", map[string]any{"liq": true}),
+ hlFillJSON("BTC", "60000.0", "0.1", 1700000001000, "0xbbb", "2", map[string]any{"liq": true}),
+ )
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 ETH event, got %d", len(events))
+ }
+ if events[0].NotionalUSD != 3000.0 {
+ t.Errorf("notional = %v, want 3000", events[0].NotionalUSD)
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_SinceFilter(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, `[%s,%s,%s]`,
+ hlFillJSON("ETH", "3000.0", "1.5", 1700000000000, "0xabc123", "1", map[string]any{"liq": true}),
+ hlFillJSON("ETH", "3000.0", "2.0", 1700000001000, "0xdef456", "2", nil),
+ hlFillJSON("ETH", "3100.0", "0.5", 1699990000000, "0xghi789", "3", map[string]any{"liq": true}),
+ )
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ events, err := h.FetchLiquidationsSince("ETH", 1699999000000)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 event (older filtered), got %d", len(events))
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_Server500(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ _, err := h.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error from 500, got nil")
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_MalformedJSON(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `not json`)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ _, err := h.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected decode error, got nil")
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_ZeroHashFallback(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, `[%s]`,
+ hlFillJSON("ETH", "3000.0", "1.0", 1700000000000,
+ "0x0000000000000000000000000000000000000000000000000000000000000000",
+ "99", map[string]any{"liq": true}),
+ )
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 event, got %d", len(events))
+ }
+ if events[0].Key != "tid:99" {
+ t.Errorf("expected tid fallback key, got %q", events[0].Key)
+ }
+}
+
+func TestHyperliquid_FetchOI_HappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&req)
+ if req["type"] == "metaAndAssetCtxs" {
+ fmt.Fprint(w, `[
+ {"universe":[{"name":"ETH"},{"name":"BTC"}]},
+ [{"openInterest":"100.5","midPx":"3000.0","markPx":"3001.0"},{"openInterest":"5.0","midPx":"60000.0","markPx":"60001.0"}]
+ ]`)
+ }
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ oi, err := h.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // 100.5 * 3000 = 301500
+ if oi < 301499 || oi > 301501 {
+ t.Errorf("OI = %v, want ~301500", oi)
+ }
+}
+
+func TestHyperliquid_FetchOI_CoinNotFound(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `[
+ {"universe":[{"name":"BTC"}]},
+ [{"openInterest":"5.0","midPx":"60000.0","markPx":"60001.0"}]
+ ]`)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ _, err := h.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for missing coin, got nil")
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_OxaHappyPath(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "/v1/hyperliquid/liquidations/") {
+ fmt.Fprint(w, `{"data":[
+ {"coin":"ETH","timestamp":"2026-08-14T12:59:09.333Z","price":"1872.3","size":"2.0","trade_id":12345},
+ {"coin":"ETH","timestamp":"2026-08-14T13:00:00.000Z","price":"1900.0","size":"0.5","trade_id":99999}
+ ],"meta":{"next_cursor":null}}`)
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL, archiveBaseURL: srv.URL, archiveAPIKey: "testkey"}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 events, got %d", len(events))
+ }
+ // 1872.3 * 2.0 = 3744.6
+ if events[0].NotionalUSD < 3744 || events[0].NotionalUSD > 3745 {
+ t.Errorf("event[0] notional = %v, want ~3744.6", events[0].NotionalUSD)
+ }
+ if events[0].Key != "oxa:12345" {
+ t.Errorf("event[0] key = %q, want oxa:12345", events[0].Key)
+ }
+ // 1900 * 0.5 = 950
+ if events[1].NotionalUSD < 949 || events[1].NotionalUSD > 951 {
+ t.Errorf("event[1] notional = %v, want ~950", events[1].NotionalUSD)
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_OxaPagination(t *testing.T) {
+ callCount := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !strings.Contains(r.URL.Path, "/v1/hyperliquid/liquidations/") {
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ callCount++
+ if r.URL.Query().Get("cursor") == "" {
+ fmt.Fprint(w, `{"data":[{"coin":"ETH","timestamp":"2026-08-14T12:00:00.000Z","price":"2000.0","size":"1.0","trade_id":1}],"meta":{"next_cursor":"abc"}}`)
+ } else {
+ fmt.Fprint(w, `{"data":[{"coin":"ETH","timestamp":"2026-08-14T13:00:00.000Z","price":"2000.0","size":"1.0","trade_id":2}],"meta":{"next_cursor":null}}`)
+ }
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL, archiveBaseURL: srv.URL, archiveAPIKey: "testkey"}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 events across 2 pages, got %d", len(events))
+ }
+ if callCount != 2 {
+ t.Errorf("expected 2 API calls (2 pages), got %d", callCount)
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_OxaError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusUnauthorized)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL, archiveBaseURL: srv.URL, archiveAPIKey: "badkey"}
+ _, err := h.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error for 401, got nil")
+ }
+}
+
+func TestHyperliquid_FetchLiquidationsSince_OxaFallsBackToVault(t *testing.T) {
+ // No archiveAPIKey → uses vault userFillsByTime path
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, `[%s]`,
+ hlFillJSON("ETH", "3000.0", "1.0", 1700000000000, "0xabc", "1", map[string]any{"liq": true}),
+ )
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL, archiveBaseURL: srv.URL, archiveAPIKey: ""}
+ events, err := h.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 event from vault fallback, got %d", len(events))
+ }
+}
+
+func TestHyperliquid_FetchOI_EmptyMidPxFallback(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `[
+ {"universe":[{"name":"ETH"}]},
+ [{"openInterest":"10.0","midPx":"","markPx":"3001.0"}]
+ ]`)
+ }))
+ defer srv.Close()
+
+ h := &Hyperliquid{infoURL: srv.URL}
+ oi, err := h.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // markPx fallback: 10 * 3001 = 30010
+ if oi < 30009 || oi > 30011 {
+ t.Errorf("OI with markPx fallback = %v, want ~30010", oi)
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_lighter.go b/harnesses/perp-liq-rate/cmd/script/source_lighter.go
new file mode 100644
index 00000000..306d3b7a
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_lighter.go
@@ -0,0 +1,213 @@
+package main
+
+// source_lighter.go — Lighter (mainnet.zklighter.elliot.ai).
+//
+// Liquidations: Lighter /api/v1/trades requires auth (HTTP 400). Data comes
+// from Coinalyze /v1/liquidation-history (symbols 0.T=ETH, 1.T=BTC), hourly
+// buckets in base asset units; multiplied by current mark_price from
+// orderBookDetails. Requires env COINALYZE_API_KEY; returns empty without it.
+// OI: GET /api/v1/orderBookDetails?filter=perp; open_interest (base units) ×
+// mark_price (USD). Market IDs resolved dynamically by symbol prefix match.
+// 404/501 marks the venue unavailable; after 3 consecutive failures the source
+// logs once and suppresses further error increments until recovery.
+
+import (
+ "errors"
+ "fmt"
+ "log"
+ "net/url"
+ "os"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ lighterBaseURL = "https://mainnet.zklighter.elliot.ai"
+ lighterMarketTTL = 2 * time.Minute
+ lighterFailLogThreshold = 3
+ czBaseURLDefault = "https://api.coinalyze.net/v1"
+)
+
+// czLighterSymbols maps asset → Coinalyze symbol for Lighter perps.
+var czLighterSymbols = map[string]string{
+ "ETH": "0.T",
+ "BTC": "1.T",
+}
+
+type lighterMarketDetail struct {
+ Symbol string `json:"symbol"`
+ MarketID int64 `json:"market_id"`
+ OpenInterest float64 `json:"open_interest"` // base units (e.g. ETH)
+ MarkPrice flexFloat `json:"mark_price"` // USD per base unit
+}
+
+// Lighter implements Source with dynamic market ID resolution and
+// consecutive-unavailability tracking.
+type Lighter struct {
+ baseURL string // defaults to lighterBaseURL
+ czBaseURL string // Coinalyze API base, defaults to czBaseURLDefault
+ czAPIKey string // from env COINALYZE_API_KEY
+
+ mu sync.Mutex
+ markets []lighterMarketDetail
+ marketsAt time.Time
+ consecUnavl int
+}
+
+// NewLighter returns the Lighter source.
+func NewLighter() *Lighter {
+ return &Lighter{
+ baseURL: lighterBaseURL,
+ czBaseURL: czBaseURLDefault,
+ czAPIKey: os.Getenv("COINALYZE_API_KEY"),
+ }
+}
+
+// fetchMarkets fetches and caches the /api/v1/orderBookDetails response.
+// On cache miss two concurrent goroutines may both fetch; the second write
+// simply overwrites the first with slightly newer data, which is harmless.
+func (l *Lighter) fetchMarkets() ([]lighterMarketDetail, error) {
+ l.mu.Lock()
+ if l.markets != nil && time.Since(l.marketsAt) < lighterMarketTTL {
+ m := l.markets
+ l.mu.Unlock()
+ return m, nil
+ }
+ l.mu.Unlock()
+
+ var resp struct {
+ OrderBookDetails []lighterMarketDetail `json:"order_book_details"`
+ }
+ if err := httpGetJSON(l.baseURL+"/api/v1/orderBookDetails?filter=perp", &resp); err != nil {
+ return nil, fmt.Errorf("lighter orderBookDetails: %w", err)
+ }
+ if len(resp.OrderBookDetails) == 0 {
+ return nil, fmt.Errorf("lighter: empty orderBookDetails")
+ }
+ l.mu.Lock()
+ l.markets = resp.OrderBookDetails
+ l.marketsAt = time.Now()
+ l.mu.Unlock()
+ return resp.OrderBookDetails, nil
+}
+
+// lighterSymbolMatchesAsset reports whether a market symbol (e.g. "ETH-USD")
+// corresponds to the requested asset (e.g. "ETH").
+func lighterSymbolMatchesAsset(symbol, asset string) bool {
+ sym := strings.ToUpper(strings.TrimSpace(symbol))
+ a := strings.ToUpper(asset)
+ return sym == a || strings.HasPrefix(sym, a+"-") || strings.HasPrefix(sym, a+"/")
+}
+
+func (l *Lighter) findMarket(asset string) (lighterMarketDetail, error) {
+ markets, err := l.fetchMarkets()
+ if err != nil {
+ return lighterMarketDetail{}, err
+ }
+ for _, m := range markets {
+ if lighterSymbolMatchesAsset(m.Symbol, asset) {
+ return m, nil
+ }
+ }
+ return lighterMarketDetail{}, fmt.Errorf("lighter: no market found for asset %q", asset)
+}
+
+// FetchLiquidationsSince returns hourly liquidation buckets from Coinalyze,
+// converted to USD using the current mark_price from orderBookDetails.
+// Returns empty if COINALYZE_API_KEY is not set.
+func (l *Lighter) FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error) {
+ market, err := l.findMarket(asset)
+ if err != nil {
+ var se *httpStatusError
+ if errors.As(err, &se) && (se.Code == 404 || se.Code == 501) {
+ return nil, l.markUnavailable(se.Code)
+ }
+ return nil, err
+ }
+
+ if l.czAPIKey == "" {
+ return nil, nil
+ }
+ czSym, ok := czLighterSymbols[asset]
+ if !ok {
+ return nil, nil
+ }
+
+ markPx := float64(market.MarkPrice)
+ if markPx == 0 {
+ return nil, nil
+ }
+
+ from := sinceMs / 1000
+ to := time.Now().Unix()
+ u := fmt.Sprintf("%s/liquidation-history?symbols=%s&interval=1hour&from=%d&to=%d&api_key=%s",
+ l.czBaseURL, url.QueryEscape(czSym), from, to, l.czAPIKey)
+
+ var resp []struct {
+ Symbol string `json:"symbol"`
+ History []struct {
+ T int64 `json:"t"` // unix seconds (bucket start)
+ L float64 `json:"l"` // long liqs (base asset)
+ S float64 `json:"s"` // short liqs (base asset)
+ } `json:"history"`
+ }
+ if err := httpGetJSON(u, &resp); err != nil {
+ return nil, fmt.Errorf("lighter coinalyze liquidations: %w", err)
+ }
+
+ var events []LiqEvent
+ for _, r := range resp {
+ for _, h := range r.History {
+ tsMs := h.T * 1000
+ if tsMs < sinceMs {
+ continue
+ }
+ total := (h.L + h.S) * markPx
+ if total == 0 {
+ continue
+ }
+ events = append(events, LiqEvent{
+ Key: fmt.Sprintf("czlighter:%s:%d", asset, h.T),
+ NotionalUSD: total,
+ TimestampMs: tsMs,
+ })
+ }
+ }
+ return events, nil
+}
+
+// markUnavailable tracks consecutive 404/501 responses; at the threshold it
+// logs once, and beyond it asks the runner to suppress counters and logs.
+func (l *Lighter) markUnavailable(status int) error {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.consecUnavl++
+ if l.consecUnavl == lighterFailLogThreshold {
+ log.Printf("[lighter] endpoint unavailable %d consecutive times (http %d); suppressing further unavailability error metrics and logs until recovery",
+ l.consecUnavl, status)
+ }
+ return &unavailableError{status: status, suppressed: l.consecUnavl > lighterFailLogThreshold}
+}
+
+func (l *Lighter) resetUnavailable() {
+ l.mu.Lock()
+ if l.consecUnavl > 0 {
+ log.Printf("[lighter] endpoint recovered after %d consecutive unavailability(ies)", l.consecUnavl)
+ }
+ l.consecUnavl = 0
+ l.mu.Unlock()
+}
+
+// FetchOI returns open_interest × mark_price from the orderBookDetails cache.
+func (l *Lighter) FetchOI(asset string) (float64, error) {
+ market, err := l.findMarket(asset)
+ if err != nil {
+ return 0, err
+ }
+ markPx := float64(market.MarkPrice)
+ if markPx == 0 {
+ return 0, fmt.Errorf("lighter: mark_price is zero for %s", asset)
+ }
+ return market.OpenInterest * markPx, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_lighter_test.go b/harnesses/perp-liq-rate/cmd/script/source_lighter_test.go
new file mode 100644
index 00000000..ea5a931c
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_lighter_test.go
@@ -0,0 +1,308 @@
+package main
+
+import (
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+// lighterTestMarket builds a market entry for the orderBookDetails mock.
+func lighterTestMarket(symbol string, marketID int64, oi float64, markPrice string) map[string]any {
+ return map[string]any{
+ "symbol": symbol,
+ "market_id": marketID,
+ "open_interest": oi,
+ "mark_price": markPrice,
+ }
+}
+
+// buildLighterOBServer creates a mock for the orderBookDetails endpoint only.
+func buildLighterOBServer(t *testing.T, markets []map[string]any) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "orderBookDetails") {
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "order_book_details": markets,
+ })
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+}
+
+func TestLighterSymbolMatchesAsset(t *testing.T) {
+ cases := []struct {
+ symbol string
+ asset string
+ want bool
+ }{
+ {"ETH-USD", "ETH", true},
+ {"ETH-USDC", "ETH", true},
+ {"ETH/USD", "ETH", true},
+ {"ETH", "ETH", true},
+ {"WETH-USD", "ETH", false},
+ {"BTC-USD", "ETH", false},
+ {"BTC-USD", "BTC", true},
+ }
+ for _, c := range cases {
+ got := lighterSymbolMatchesAsset(c.symbol, c.asset)
+ if got != c.want {
+ t.Errorf("lighterSymbolMatchesAsset(%q, %q) = %v, want %v", c.symbol, c.asset, got, c.want)
+ }
+ }
+}
+
+// FetchLiquidationsSince now returns empty (trades endpoint requires auth).
+
+func TestLighter_FetchLiquidationsSince_ReturnsEmpty(t *testing.T) {
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 42.0, "1878.0")}
+ srv := buildLighterOBServer(t, markets)
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ events, err := l.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events, got %d", len(events))
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_404MarketList(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ _, err := l.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error for 404")
+ }
+ if !errors.Is(err, ErrVenueUnavailable) {
+ t.Errorf("expected ErrVenueUnavailable, got %v", err)
+ }
+ if l.consecUnavl != 1 {
+ t.Errorf("consecUnavl = %d, want 1", l.consecUnavl)
+ }
+ var ue *unavailableError
+ if errors.As(err, &ue) && ue.suppressed {
+ t.Error("first occurrence should not be suppressed")
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_404Repeated(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ var lastErr error
+ for i := 0; i < 4; i++ {
+ _, lastErr = l.FetchLiquidationsSince("ETH", 0)
+ }
+ if lastErr == nil {
+ t.Fatal("expected error")
+ }
+ var ue *unavailableError
+ if !errors.As(lastErr, &ue) {
+ t.Fatalf("expected unavailableError, got %T", lastErr)
+ }
+ if !ue.suppressed {
+ t.Error("4th occurrence should be suppressed (threshold is 3)")
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_RecoveryAfterStreak(t *testing.T) {
+ failCount := 0
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 42.0, "1878.0")}
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if failCount < 3 {
+ failCount++
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"order_book_details": markets})
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ for i := 0; i < 3; i++ {
+ l.FetchLiquidationsSince("ETH", 0) //nolint
+ }
+ if l.consecUnavl != 3 {
+ t.Fatalf("consecUnavl = %d, want 3 before recovery", l.consecUnavl)
+ }
+
+ _, err := l.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("expected recovery, got error: %v", err)
+ }
+ // resetUnavailable is not called in FetchLiquidationsSince (no trades fetch),
+ // so consecUnavl stays at 3 after recovery — check that the error is gone.
+ if err != nil {
+ t.Errorf("post-recovery error: %v", err)
+ }
+}
+
+func TestLighter_FetchOI_HappyPath(t *testing.T) {
+ markets := []map[string]any{
+ lighterTestMarket("ETH-USD", 0, 42279.85, "1878.2"),
+ }
+ srv := buildLighterOBServer(t, markets)
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ oi, err := l.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // 42279.85 * 1878.2 ≈ 79,406,624
+ if oi < 79000000 || oi > 80000000 {
+ t.Errorf("OI = %v, want ~79.4M", oi)
+ }
+}
+
+func TestLighter_FetchOI_ZeroMarkPrice(t *testing.T) {
+ markets := []map[string]any{
+ lighterTestMarket("ETH-USD", 0, 93.0, "0"),
+ }
+ srv := buildLighterOBServer(t, markets)
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ _, err := l.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for zero mark_price, got nil")
+ }
+}
+
+func TestLighter_FetchOI_MarketNotFound(t *testing.T) {
+ markets := []map[string]any{
+ lighterTestMarket("SOL-USD", 2, 10.0, "100.0"),
+ }
+ srv := buildLighterOBServer(t, markets)
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ _, err := l.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for unknown asset, got nil")
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_CoinalyzeNoKey(t *testing.T) {
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 42.0, "1878.0")}
+ srv := buildLighterOBServer(t, markets)
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL, czBaseURL: srv.URL, czAPIKey: ""}
+ events, err := l.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events without api key, got %d", len(events))
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_CoinalyzeHappyPath(t *testing.T) {
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 42.0, "2000.0")}
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "orderBookDetails") {
+ _ = json.NewEncoder(w).Encode(map[string]any{"order_book_details": markets})
+ return
+ }
+ if strings.Contains(r.URL.Path, "liquidation-history") {
+ _ = json.NewEncoder(w).Encode([]map[string]any{
+ {"symbol": "0.T", "history": []map[string]any{
+ {"t": int64(1786233600), "l": 3.0, "s": 2.0},
+ {"t": int64(1786237200), "l": 0.0, "s": 0.0}, // zero bucket, skipped
+ }},
+ })
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL, czBaseURL: srv.URL, czAPIKey: "testkey"}
+ events, err := l.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 event (zero bucket skipped), got %d", len(events))
+ }
+ // (3 + 2) ETH * 2000 = 10000
+ if events[0].NotionalUSD < 9999 || events[0].NotionalUSD > 10001 {
+ t.Errorf("notional = %v, want ~10000", events[0].NotionalUSD)
+ }
+ if events[0].Key != "czlighter:ETH:1786233600" {
+ t.Errorf("key = %q, want czlighter:ETH:1786233600", events[0].Key)
+ }
+}
+
+func TestLighter_FetchLiquidationsSince_CoinalyzeSinceFilter(t *testing.T) {
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 10.0, "1000.0")}
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "orderBookDetails") {
+ _ = json.NewEncoder(w).Encode(map[string]any{"order_book_details": markets})
+ return
+ }
+ if strings.Contains(r.URL.Path, "liquidation-history") {
+ _ = json.NewEncoder(w).Encode([]map[string]any{
+ {"symbol": "0.T", "history": []map[string]any{
+ {"t": int64(1000), "l": 5.0, "s": 0.0}, // before sinceMs (2000000ms), skipped
+ {"t": int64(3000), "l": 2.0, "s": 1.0}, // after sinceMs (2000000ms), kept
+ }},
+ })
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL, czBaseURL: srv.URL, czAPIKey: "testkey"}
+ events, err := l.FetchLiquidationsSince("ETH", 2000000) // sinceMs = 2000 unix seconds in ms
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("expected 1 event (old bucket filtered), got %d", len(events))
+ }
+}
+
+func TestLighter_MarketsCache(t *testing.T) {
+ callCount := 0
+ markets := []map[string]any{lighterTestMarket("ETH-USD", 0, 42.0, "1878.0")}
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "orderBookDetails") {
+ callCount++
+ _ = json.NewEncoder(w).Encode(map[string]any{"order_book_details": markets})
+ return
+ }
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ l := &Lighter{baseURL: srv.URL}
+ if _, err := l.FetchOI("ETH"); err != nil {
+ t.Fatalf("first FetchOI error: %v", err)
+ }
+ if _, err := l.FetchOI("ETH"); err != nil {
+ t.Fatalf("second FetchOI error: %v", err)
+ }
+ if callCount != 1 {
+ t.Errorf("orderBookDetails called %d times, want 1 (cache hit on second call)", callCount)
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_paradex.go b/harnesses/perp-liq-rate/cmd/script/source_paradex.go
new file mode 100644
index 00000000..f6bce148
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_paradex.go
@@ -0,0 +1,130 @@
+package main
+
+// source_paradex.go — Paradex.
+//
+// Liquidations: GET /v1/trades?market=X&start_at=&end_at=
+// &page_size=100, following the "next" cursor; keep rows whose trade_type
+// equals "LIQUIDATION". VERIFY: exact trade_type value — the field was added
+// in Paradex v1.38; the assumed value is "LIQUIDATION". Notional = size×price.
+// OI: GET /v1/markets/summary?market=X → open_interest. VERIFY: unit
+// (base-asset vs USD) by comparing BTC vs ETH order of magnitude at runtime.
+
+import (
+ "fmt"
+ "net/url"
+ "time"
+)
+
+const (
+ paradexBaseURL = "https://api.prod.paradex.trade/v1"
+ paradexPageSize = 100
+ paradexMaxPages = 20
+)
+
+var paradexMarkets = map[string]string{
+ "ETH": "ETH-USD-PERP",
+ "BTC": "BTC-USD-PERP",
+}
+
+// Paradex implements Source. It is stateless and shared across assets.
+type Paradex struct {
+ baseURL string // defaults to paradexBaseURL
+}
+
+// NewParadex returns the Paradex source.
+func NewParadex() *Paradex { return &Paradex{baseURL: paradexBaseURL} }
+
+type paradexTrade struct {
+ ID string `json:"id"`
+ Size string `json:"size"`
+ Price string `json:"price"`
+ CreatedAt int64 `json:"created_at"` // unix ms
+ TradeType string `json:"trade_type"` // "FILL", "LIQUIDATION", etc.
+}
+
+type paradexTradesResp struct {
+ Results []paradexTrade `json:"results"`
+ Next *string `json:"next"`
+}
+
+// FetchLiquidationsSince pages the public trade tape and keeps LIQUIDATION rows.
+func (p *Paradex) FetchLiquidationsSince(asset string, sinceMs int64) ([]LiqEvent, error) {
+ market, ok := paradexMarkets[asset]
+ if !ok {
+ return nil, fmt.Errorf("paradex: unsupported asset %q", asset)
+ }
+
+ var events []LiqEvent
+ cursor := ""
+ endMs := time.Now().UnixMilli()
+
+ for page := 0; page < paradexMaxPages; page++ {
+ u := fmt.Sprintf("%s/trades?market=%s&start_at=%d&end_at=%d&page_size=%d",
+ p.baseURL, url.QueryEscape(market), sinceMs, endMs, paradexPageSize)
+ if cursor != "" {
+ u += "&cursor=" + url.QueryEscape(cursor)
+ }
+ var resp paradexTradesResp
+ if err := httpGetJSON(u, &resp); err != nil {
+ return nil, fmt.Errorf("paradex trades: %w", err)
+ }
+ for _, t := range resp.Results {
+ if t.TradeType != "LIQUIDATION" {
+ continue
+ }
+ if t.CreatedAt < sinceMs {
+ continue
+ }
+ size, err := parseF(t.Size)
+ if err != nil {
+ return nil, fmt.Errorf("paradex size: %w", err)
+ }
+ price, err := parseF(t.Price)
+ if err != nil {
+ return nil, fmt.Errorf("paradex price: %w", err)
+ }
+ key := t.ID
+ if key == "" {
+ key = fmt.Sprintf("%s:%d:%s:%s", market, t.CreatedAt, t.Size, t.Price)
+ }
+ events = append(events, LiqEvent{
+ Key: "paradex:" + key,
+ NotionalUSD: size * price,
+ TimestampMs: t.CreatedAt,
+ })
+ }
+ if resp.Next == nil || *resp.Next == "" || *resp.Next == "null" || len(resp.Results) == 0 {
+ break
+ }
+ cursor = *resp.Next
+ }
+ return events, nil
+}
+
+// FetchOI returns open interest in USD from the market summary.
+// open_interest is in base-asset units (ETH, BTC); multiply by mark_price.
+func (p *Paradex) FetchOI(asset string) (float64, error) {
+ market, ok := paradexMarkets[asset]
+ if !ok {
+ return 0, fmt.Errorf("paradex: unsupported asset %q", asset)
+ }
+ var resp struct {
+ Results []struct {
+ OpenInterest flexFloat `json:"open_interest"` // base units
+ MarkPrice flexFloat `json:"mark_price"` // USD per base unit
+ } `json:"results"`
+ }
+ u := fmt.Sprintf("%s/markets/summary?market=%s", p.baseURL, url.QueryEscape(market))
+ if err := httpGetJSON(u, &resp); err != nil {
+ return 0, fmt.Errorf("paradex markets/summary: %w", err)
+ }
+ if len(resp.Results) == 0 {
+ return 0, fmt.Errorf("paradex markets/summary: empty results for %s", market)
+ }
+ r := resp.Results[0]
+ markPx := float64(r.MarkPrice)
+ if markPx == 0 {
+ return 0, fmt.Errorf("paradex markets/summary: mark_price is zero for %s", market)
+ }
+ return float64(r.OpenInterest) * markPx, nil
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/source_paradex_test.go b/harnesses/perp-liq-rate/cmd/script/source_paradex_test.go
new file mode 100644
index 00000000..bb9ee9e3
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/source_paradex_test.go
@@ -0,0 +1,184 @@
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func makeParadexTrade(id string, ts int64, tradeType string) map[string]any {
+ return map[string]any{
+ "id": id,
+ "size": "1.5",
+ "price": "3000.0",
+ "created_at": ts,
+ "trade_type": tradeType,
+ }
+}
+
+func TestParadex_FetchLiquidationsSince_SinglePage(t *testing.T) {
+ now := int64(1700000000000)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "results": []map[string]any{
+ makeParadexTrade("f1", now, "LIQUIDATION"),
+ makeParadexTrade("f2", now+1000, "LIQUIDATION"),
+ makeParadexTrade("f3", now+2000, "FILL"), // filtered out
+ },
+ "next": nil,
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ events, err := p.FetchLiquidationsSince("ETH", now-1000)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 2 {
+ t.Fatalf("expected 2 LIQUIDATION events, got %d", len(events))
+ }
+ // 1.5 * 3000 = 4500
+ if events[0].NotionalUSD < 4499 || events[0].NotionalUSD > 4501 {
+ t.Errorf("notional = %v, want ~4500", events[0].NotionalUSD)
+ }
+}
+
+func TestParadex_FetchLiquidationsSince_Pagination(t *testing.T) {
+ now := int64(1700000000000)
+ callCount := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ callCount++
+ if callCount == 1 {
+ next := "tok2"
+ resp := map[string]any{
+ "results": []map[string]any{
+ makeParadexTrade("f1", now, "LIQUIDATION"),
+ makeParadexTrade("f2", now+1000, "LIQUIDATION"),
+ },
+ "next": &next,
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ return
+ }
+ resp := map[string]any{
+ "results": []map[string]any{
+ makeParadexTrade("f3", now+2000, "LIQUIDATION"),
+ },
+ "next": nil,
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ events, err := p.FetchLiquidationsSince("ETH", now-1000)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 3 {
+ t.Fatalf("expected 3 events across 2 pages, got %d", len(events))
+ }
+ if callCount != 2 {
+ t.Errorf("expected 2 server calls for pagination, got %d", callCount)
+ }
+}
+
+func TestParadex_FetchLiquidationsSince_NonLiquidationFiltered(t *testing.T) {
+ now := int64(1700000000000)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "results": []map[string]any{
+ makeParadexTrade("t1", now, "FILL"),
+ makeParadexTrade("t2", now+1000, "FILL"),
+ },
+ "next": nil,
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ events, err := p.FetchLiquidationsSince("ETH", 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if len(events) != 0 {
+ t.Fatalf("expected 0 events (all FILL type), got %d", len(events))
+ }
+}
+
+func TestParadex_FetchLiquidationsSince_ServerError(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ _, err := p.FetchLiquidationsSince("ETH", 0)
+ if err == nil {
+ t.Fatal("expected error from 500, got nil")
+ }
+}
+
+func TestParadex_FetchOI_HappyPath(t *testing.T) {
+ // open_interest is in base units (ETH); OI_USD = open_interest * mark_price
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "results": []map[string]any{
+ {
+ "open_interest": "25.0", // 25 ETH
+ "mark_price": "3000.0", // $3000/ETH
+ },
+ },
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ oi, err := p.FetchOI("ETH")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // 25 * 3000 = 75000
+ if oi < 74999 || oi > 75001 {
+ t.Errorf("OI = %v, want ~75000", oi)
+ }
+}
+
+func TestParadex_FetchOI_ZeroMarkPriceFails(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "results": []map[string]any{
+ {"open_interest": "25.0", "mark_price": "0"},
+ },
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ _, err := p.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for zero mark_price, got nil")
+ }
+}
+
+func TestParadex_FetchOI_EmptyResults(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp := map[string]any{
+ "results": []map[string]any{},
+ }
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+ defer srv.Close()
+
+ p := &Paradex{baseURL: srv.URL}
+ _, err := p.FetchOI("ETH")
+ if err == nil {
+ t.Fatal("expected error for empty results, got nil")
+ }
+}
diff --git a/harnesses/perp-liq-rate/cmd/script/window.go b/harnesses/perp-liq-rate/cmd/script/window.go
new file mode 100644
index 00000000..ef17aa26
--- /dev/null
+++ b/harnesses/perp-liq-rate/cmd/script/window.go
@@ -0,0 +1,130 @@
+package main
+
+// window.go — thread-safe 24h sliding window of liquidation notionals plus
+// the dedup SeenSet that is pruned alongside it.
+
+import (
+ "sync"
+ "time"
+)
+
+type windowEntry struct {
+ tsMs int64
+ notional float64
+}
+
+// SlidingWindow accumulates (unix_ms, notional_usd) events and answers the
+// rolling sum over its span. All methods are safe for concurrent use.
+type SlidingWindow struct {
+ mu sync.Mutex
+ entries []windowEntry
+ span time.Duration
+ firstAt time.Time // wall time of the first tick that touched this window
+}
+
+// NewSlidingWindow returns a window covering the given span (24h here).
+func NewSlidingWindow(span time.Duration) *SlidingWindow {
+ return &SlidingWindow{span: span}
+}
+
+// Add appends one event to the window.
+func (w *SlidingWindow) Add(tsMs int64, notionalUSD float64) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.entries = append(w.entries, windowEntry{tsMs: tsMs, notional: notionalUSD})
+}
+
+// MarkTick records the first time a tick ran against this window; used by
+// IsWarm to decide when a full span of data has been observed.
+func (w *SlidingWindow) MarkTick(now time.Time) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.firstAt.IsZero() {
+ w.firstAt = now
+ }
+}
+
+// Prune drops entries older than span relative to nowMs.
+func (w *SlidingWindow) Prune(nowMs int64) {
+ cutoff := nowMs - w.span.Milliseconds()
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ kept := w.entries[:0]
+ for _, e := range w.entries {
+ if e.tsMs >= cutoff {
+ kept = append(kept, e)
+ }
+ }
+ // Zero the tail so pruned entries do not linger in the backing array.
+ for i := len(kept); i < len(w.entries); i++ {
+ w.entries[i] = windowEntry{}
+ }
+ w.entries = kept
+}
+
+// Sum returns the total notional currently inside the window.
+func (w *SlidingWindow) Sum() float64 {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ total := 0.0
+ for _, e := range w.entries {
+ total += e.notional
+ }
+ return total
+}
+
+// Len returns the number of events currently inside the window.
+func (w *SlidingWindow) Len() int {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ return len(w.entries)
+}
+
+// IsWarm reports whether a full window span has elapsed since the first tick,
+// i.e. whether the 24h sum is trustworthy.
+func (w *SlidingWindow) IsWarm(now time.Time) bool {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ return !w.firstAt.IsZero() && now.Sub(w.firstAt) >= w.span
+}
+
+// SeenSet tracks deduplication keys together with the event timestamp so
+// stale keys can be pruned alongside the sliding window.
+type SeenSet struct {
+ mu sync.Mutex
+ m map[string]int64
+}
+
+// NewSeenSet returns an empty SeenSet.
+func NewSeenSet() *SeenSet {
+ return &SeenSet{m: make(map[string]int64)}
+}
+
+// Add records key with its timestamp and reports whether the key was new.
+func (s *SeenSet) Add(key string, tsMs int64) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if _, ok := s.m[key]; ok {
+ return false
+ }
+ s.m[key] = tsMs
+ return true
+}
+
+// Prune deletes keys whose event timestamp is older than cutoffMs.
+func (s *SeenSet) Prune(cutoffMs int64) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for k, ts := range s.m {
+ if ts < cutoffMs {
+ delete(s.m, k)
+ }
+ }
+}
+
+// Len returns the number of tracked keys.
+func (s *SeenSet) Len() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return len(s.m)
+}
diff --git a/harnesses/perp-liq-rate/go.mod b/harnesses/perp-liq-rate/go.mod
new file mode 100644
index 00000000..c9b16fce
--- /dev/null
+++ b/harnesses/perp-liq-rate/go.mod
@@ -0,0 +1,17 @@
+module github.com/mobula/perp-liq-rate
+
+go 1.24
+
+require github.com/prometheus/client_golang v1.20.5
+
+require (
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/klauspost/compress v1.17.9 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/prometheus/client_model v0.6.1 // indirect
+ github.com/prometheus/common v0.55.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
+ golang.org/x/sys v0.22.0 // indirect
+ google.golang.org/protobuf v1.34.2 // indirect
+)
diff --git a/harnesses/perp-liq-rate/go.sum b/harnesses/perp-liq-rate/go.sum
new file mode 100644
index 00000000..d5318cf8
--- /dev/null
+++ b/harnesses/perp-liq-rate/go.sum
@@ -0,0 +1,24 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=