Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion packages/stats/core/src/domain/inference.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test"
import { toGeoAggregate, toModelAggregate, toProviderAggregate } from "./inference"
import { buildStatsQuery, toGeoAggregate, toModelAggregate, toProviderAggregate } from "./inference"
import { modelAuthor, normalizeInferenceModel, statModel, statProvider } from "./model-normalization"

describe("inference stat normalization", () => {
Expand Down Expand Up @@ -38,6 +38,22 @@ describe("inference stat normalization", () => {
expect(statProvider("unknown", "", "custom-provider")).toBe("custom-provider")
})

test("merges union alpha routes without exposing the provider", () => {
;["opencode-go/union-alpha", "opencode/union-alpha"].forEach((model) => {
expect(statModel(model, "")).toBe("union-alpha")
expect(statProvider(model, "gpt-test-model", "test-provider")).toBe("unknown")
expect(
toModelAggregate({ ...aggregate(model, "test-provider"), provider_model: "gpt-test-model" }),
).toMatchObject([{ model: "union-alpha", provider: "unknown", requests: 1 }])
expect(toProviderAggregate(aggregate(model, "test-provider"))).toMatchObject([
{ provider: "unknown", requests: 1 },
])
expect(toGeoAggregate({ ...aggregate(model, "test-provider"), country: "US" })).toMatchObject([
{ model: "union-alpha", provider: "unknown", country: "US", requests: 1 },
])
})
})

test("merges renamed models under their current name", () => {
expect(statModel("deepseek-flash", "")).toBe("deepseek-v4.1-flash")
expect(statModel("x-preview-f", "")).toBe("ox-alpha")
Expand Down Expand Up @@ -94,6 +110,20 @@ describe("inference stat normalization", () => {
}),
).toMatchObject([{ period_key: "2026-W20" }])
})

test("normalizes union alpha routes in stat queries", () => {
process.env.SST_RESOURCE_InferenceEvent = JSON.stringify({
catalog: "inference",
database: "events",
table: "generation",
})
process.env.SST_RESOURCE_StatsSyncConfig = JSON.stringify({ dataset: "zen" })
const query = buildStatsQuery(new Date("2026-05-20T00:00:00.000Z"), new Date("2026-05-21T00:00:00.000Z"))

expect(query).toContain("WHEN lower(model) = 'opencode-go/union-alpha' THEN 'union-alpha'")
expect(query).toContain("WHEN lower(model) = 'opencode/union-alpha' THEN 'union-alpha'")
expect(query).toContain("WHEN lower(model) IN ('omen-alpha', 'union-alpha') THEN 'unknown'")
})
})

function aggregate(model: string, provider: string) {
Expand Down
2 changes: 2 additions & 0 deletions packages/stats/core/src/domain/inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
MODEL_AUTHOR_RULES,
MODEL_NAME_ALIASES,
RETIRED_STAT_PROVIDERS,
STEALTH_MODELS,
statModel,
statProvider,
} from "./model-normalization"
Expand Down Expand Up @@ -263,6 +264,7 @@ ${Object.entries(MODEL_NAME_ALIASES)

function statProviderSql(model: string, providerModel: string, provider: string) {
return `CASE
WHEN lower(${model}) IN (${[...STEALTH_MODELS].map(sqlString).join(", ")}) THEN 'unknown'
${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${providerModel}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")}
${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${model}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")}
WHEN ${provider} <> '' AND lower(${provider}) NOT IN (${RETIRED_STAT_PROVIDERS.map(sqlString).join(", ")}) THEN ${provider}
Expand Down
8 changes: 7 additions & 1 deletion packages/stats/core/src/domain/model-normalization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@ export const MODEL_AUTHOR_RULES = [
{ match: "qwen", author: "qwen" },
] as const
export const EXCLUDED_MODELS = new Set(["alpha-gpt-next"])
export const STEALTH_MODELS = new Set(["omen-alpha", "union-alpha"])
export const MODEL_NAME_ALIASES: Record<string, string> = {
"deepseek-flash": "deepseek-v4.1-flash",
"opencode-go/union-alpha": "union-alpha",
"opencode/union-alpha": "union-alpha",
"x-preview-f": "ox-alpha",
"xiaomi/mimo-v2.5": "mimo-v2.5",
}
Expand Down Expand Up @@ -45,7 +48,10 @@ export function statProvider(
providerModel: string | undefined,
provider: string | undefined,
) {
const modelAuthorValue = modelAuthor(statModel(model, providerModel))
const normalized = statModel(model, providerModel)
if (STEALTH_MODELS.has(normalized.toLowerCase())) return "unknown"

const modelAuthorValue = modelAuthor(normalized)
if (!modelAuthorValue) return undefined

const providerModelAuthor = modelAuthor(providerModel)
Expand Down
Loading