From da04c41de21f53301cfc55a0a90d21bd4e9787fb Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:23:29 -0700 Subject: [PATCH] feat(review): expose real BYOK token/cost usage in ops stats Closes #3925. The hosted gittensory-api Worker's ONLY possible AI activity is a maintainer's own BYOK call (the legacy Workers-AI binding path is retired; env.AI is undefined there) -- but nothing previously read back the real token/cost columns migration 0109 added to ai_usage_events for the hosted deployment. The one dashboard built for this (orb-ai-usage.json) is wired exclusively to self-host's own local reporting-export SQLite mirror, which cannot see the hosted D1. Adds sumByokAiUsageForRepoSince (real, not estimated, tokens + cost over a trailing 24h window) and wires it into the existing /v1/internal/ops/stats payload as a new byokUsage field per repo. Currently zero repos have a BYOK key configured, so this reports all zeros today -- the point is that a future burn is now observable at all, rather than requiring an ad hoc manual D1 query. --- src/db/repositories.ts | 43 ++++++++++++++++++++++++++++++++++++++ src/review/ops-wire.ts | 16 ++++++++++++-- test/unit/ops-wire.test.ts | 42 +++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 27ae191ff5..1f8e71508f 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -3244,6 +3244,49 @@ export async function countByokAiEventsForRepoSince(env: Env, repoFullName: stri return Number(row?.total ?? 0); } +/** + * #hosted-ai-usage-observability: the ONLY AI activity the HOSTED gittensory-api Worker can ever have is a + * maintainer's own BYOK call (the legacy Workers-AI-binding path is retired; `env.AI` is undefined there) -- + * yet nothing previously read back the real token/cost columns migration 0109 added to `ai_usage_events` for + * the hosted deployment specifically (the one dashboard built for this, orb-ai-usage.json, is wired + * exclusively to self-host's own local reporting-export SQLite mirror and cannot see the hosted D1 at all). + * Real, not estimated: sums the actual provider-reported input/output/total tokens and cost_usd, not the + * estimatedNeurons quota-proxy sumAiEstimatedNeuronsSince already tracks. + */ +export async function sumByokAiUsageForRepoSince( + env: Env, + repoFullName: string, + sinceIso: string, +): Promise<{ calls: number; inputTokens: number; outputTokens: number; totalTokens: number; costUsd: number }> { + const db = getDb(env.DB); + const [row] = await db + .select({ + calls: sql`count(*)`, + inputTokens: sql`coalesce(sum(${aiUsageEvents.inputTokens}), 0)`, + outputTokens: sql`coalesce(sum(${aiUsageEvents.outputTokens}), 0)`, + totalTokens: sql`coalesce(sum(${aiUsageEvents.totalTokens}), 0)`, + costUsd: sql`coalesce(sum(${aiUsageEvents.costUsd}), 0)`, + }) + .from(aiUsageEvents) + .where( + and( + gte(aiUsageEvents.createdAt, sinceIso), + eq(aiUsageEvents.status, "ok"), + sql`${aiUsageEvents.model} like 'byok:%'`, + sql`json_extract(${aiUsageEvents.metadataJson}, '$.repoFullName') = ${repoFullName}`, + ), + ); + /* v8 ignore next -- SQL aggregate sum/count always returns one row; fallback protects D1 driver anomalies. */ + if (!row) return { calls: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 }; + return { + calls: Number(row.calls), + inputTokens: Number(row.inputTokens), + outputTokens: Number(row.outputTokens), + totalTokens: Number(row.totalTokens), + costUsd: Number(row.costUsd), + }; +} + export async function upsertContributorScoringProfile(env: Env, profile: ContributorScoringProfileRecord): Promise { const db = getDb(env.DB); await db diff --git a/src/review/ops-wire.ts b/src/review/ops-wire.ts index f218fecee2..a3155e980d 100644 --- a/src/review/ops-wire.ts +++ b/src/review/ops-wire.ts @@ -30,7 +30,7 @@ // `override_audit` D1 tables (none of which exist in gittensory's migrations yet) plus a careful soak/promote // design. This module is READ-ONLY observability: it reports drift; it never changes what blocks a live PR. -import { findHottestInconclusiveReviewTargetForRepo, findHottestReviewTargetForRepo, listRepositories } from "../db/repositories"; +import { findHottestInconclusiveReviewTargetForRepo, findHottestReviewTargetForRepo, listRepositories, sumByokAiUsageForRepoSince } from "../db/repositories"; import { isAgentConfigured } from "../settings/autonomy"; import { resolveRepositorySettings } from "../settings/repository-settings"; import { loadGatePrecisionReport, type GatePrecisionReport } from "../services/gate-precision"; @@ -69,6 +69,11 @@ const REVIEW_BURST_WINDOW_HOURS = 2; * and more anomalous than a publish burst (normal iteration never produces repeated INCONCLUSIVE calls). */ const REVIEW_FAILURE_BURST_THRESHOLD = 3; +/** #hosted-ai-usage-observability: the trailing window computeOpsStats' byokUsage rollup covers. Wider than the + * burst windows above on purpose -- this is a spend-visibility figure an operator checks periodically, not a + * same-tick anomaly to alert on. */ +const BYOK_USAGE_WINDOW_HOURS = 24; + /** One repo's outcome reports + the repo it covers — the input to the pure anomaly detector. `reviewBurst` and * `reviewFailureBurst` are optional so existing snapshot-fixture tests need not be touched; absent/null means * "not computed", not "healthy" -- the caller (runOpsAlerts/computeOpsStats) always populates both today. */ @@ -209,6 +214,10 @@ export interface OpsStatsRepoRow { recommendations: { total: number; positive: number; negative: number; pending: number; positiveRate: number | null }; /** The active anomaly lines for this repo (same as the cron alert), so the dashboard can flag drift. */ anomalies: string[]; + /** #hosted-ai-usage-observability: real (not estimated) BYOK token/cost usage over the trailing + * BYOK_USAGE_WINDOW_HOURS -- the only AI activity the hosted Worker can ever have (the legacy Workers-AI + * binding path is retired). Previously nothing exposed this for the hosted deployment at all. */ + byokUsage: { calls: number; inputTokens: number; outputTokens: number; totalTokens: number; costUsd: number }; } export interface OpsStatsPayload { @@ -224,13 +233,15 @@ export async function computeOpsStats(env: Env): Promise { const repos = await opsScanRepos(env); const rows: OpsStatsRepoRow[] = []; const reviewBurstSinceIso = new Date(Date.now() - REVIEW_BURST_WINDOW_HOURS * 60 * 60 * 1000).toISOString(); + const byokUsageSinceIso = new Date(Date.now() - BYOK_USAGE_WINDOW_HOURS * 60 * 60 * 1000).toISOString(); for (const repoFullName of repos) { try { - const [gatePrecision, calibration, reviewBurst, reviewFailureBurst] = await Promise.all([ + const [gatePrecision, calibration, reviewBurst, reviewFailureBurst, byokUsage] = await Promise.all([ loadGatePrecisionReport(env, repoFullName), buildRepoOutcomeCalibration(env, repoFullName), findHottestReviewTargetForRepo(env, repoFullName, reviewBurstSinceIso), findHottestInconclusiveReviewTargetForRepo(env, repoFullName, reviewBurstSinceIso), + sumByokAiUsageForRepoSince(env, repoFullName, byokUsageSinceIso), ]); rows.push({ repoFullName, @@ -246,6 +257,7 @@ export async function computeOpsStats(env: Env): Promise { }, recommendations: calibration.recommendations, anomalies: detectOutcomeAnomalies({ repoFullName, gatePrecision, calibration, reviewBurst, reviewFailureBurst }), + byokUsage, }); } catch { /* a per-repo failure must not blank the whole feed */ diff --git a/test/unit/ops-wire.test.ts b/test/unit/ops-wire.test.ts index 282574867e..3ee41faa8c 100644 --- a/test/unit/ops-wire.test.ts +++ b/test/unit/ops-wire.test.ts @@ -336,6 +336,48 @@ describe("computeOpsStats — cross-repo outcome aggregate", () => { // Privacy: aggregate only — never actor logins / trust internals. expect(JSON.stringify(payload)).not.toMatch(/login|actor|reward|payout|trust|wallet|hotkey|credibility/i); }); + + it("rolls up real BYOK token/cost usage over the trailing window (#hosted-ai-usage-observability)", async () => { + const env = createTestEnv(); + await seedRegisteredRepo(env, "owner/repo"); + await recordAiUsageEvent(env, { + feature: "ai_review_pr", + model: "byok:anthropic", + provider: "anthropic", + status: "ok", + estimatedNeurons: 0, + inputTokens: 1000, + outputTokens: 500, + totalTokens: 1500, + costUsd: 0.045, + metadata: { repoFullName: "owner/repo", pullNumber: 7, inconclusive: false }, + }); + await recordAiUsageEvent(env, { + feature: "ai_review_pr", + model: "byok:anthropic", + provider: "anthropic", + status: "ok", + estimatedNeurons: 0, + inputTokens: 2000, + outputTokens: 800, + totalTokens: 2800, + costUsd: 0.09, + metadata: { repoFullName: "owner/repo", pullNumber: 8, inconclusive: false }, + }); + + const payload = await computeOpsStats(env); + const row = payload.repos.find((r) => r.repoFullName === "owner/repo"); + expect(row?.byokUsage).toEqual({ calls: 2, inputTokens: 3000, outputTokens: 1300, totalTokens: 4300, costUsd: 0.135 }); + }); + + it("reports zero BYOK usage for a repo with no BYOK calls in the window", async () => { + const env = createTestEnv(); + await seedRegisteredRepo(env, "owner/repo"); + + const payload = await computeOpsStats(env); + const row = payload.repos.find((r) => r.repoFullName === "owner/repo"); + expect(row?.byokUsage).toEqual({ calls: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0, costUsd: 0 }); + }); }); describe("GET /v1/internal/ops/stats — bearer-gated, flag-gated endpoint", () => {