From 252e87e4ece8396e206953bef89b24ee05592a31 Mon Sep 17 00:00:00 2001 From: Michael Ryaboy Date: Wed, 23 Sep 2026 13:30:45 -0700 Subject: [PATCH] Expose classification token usage and customer pricing --- .github/workflows/check.yml | 7 ++++ .github/workflows/deploy.yml | 7 ++++ e2e/usage.mjs | 64 +++++++++++++++++++++++++++++ package.json | 3 +- src/classification-usage.ts | 41 ++++++++++++++++++ src/http/classification.ts | 17 +++++++- src/http/spending-classification.ts | 10 ++++- src/index.ts | 10 ++++- src/openapi.ts | 33 +++++++++++++++ tests/spending.e2e.test.ts | 3 ++ 10 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 e2e/usage.mjs create mode 100644 src/classification-usage.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6bb1b29..96e6d7c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -44,6 +44,13 @@ jobs: run: npm run build - name: Verify spending through the built Worker and real Durable Objects run: npm run test:e2e:spending + - name: Verify classification usage responses + run: npm run test:e2e:usage + - name: Save classification usage evidence + uses: actions/upload-artifact@v4 + with: + name: usage-e2e + path: captures/usage-e2e.json - name: Save spending verification evidence uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 284bd52..280b703 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -94,6 +94,13 @@ jobs: run: npm run build - name: Verify spending through the built Worker and real Durable Objects run: npm run test:e2e:spending + - name: Verify classification usage responses + run: npm run test:e2e:usage + - name: Save classification usage evidence + uses: actions/upload-artifact@v4 + with: + name: usage-e2e + path: captures/usage-e2e.json - name: Save spending verification evidence uses: actions/upload-artifact@v4 with: diff --git a/e2e/usage.mjs b/e2e/usage.mjs new file mode 100644 index 0000000..de54706 --- /dev/null +++ b/e2e/usage.mjs @@ -0,0 +1,64 @@ +import { Miniflare, convertV4MiniflareOptions, Response as WorkerResponse } from 'miniflare'; +import { mkdir, writeFile, readdir } from 'node:fs/promises'; +import assert from 'node:assert/strict'; + +const report = { runtime: 'built Worker in workerd', upstream: 'deterministic HTTP fixtures; no live inference', results: [] }; +let missing = false; +const modules = (await readdir('dist/server', { recursive: true })).filter(p => p.endsWith('.js')).sort((a, b) => a === 'index.js' ? -1 : b === 'index.js' ? 1 : a.localeCompare(b)).map(p => ({ type: 'ESModule', path: `dist/server/${p}` })); +const mf = new Miniflare(convertV4MiniflareOptions({ workers: [{ name: 'usage', modules, modulesRoot: 'dist/server', compatibilityDate: '2026-08-01', compatibilityFlags: ['nodejs_compat'], + durableObjects: { FREE_BUDGET: { className: 'FreeBudget', useSQLite: true }, LIMITER: { className: 'RateLimiter', useSQLite: true } }, kvNamespaces: ['STATS'], + bindings: { TYPESAFE_API_KEY: 'fixture', OPENROUTER_API_KEY: 'fixture', PRIVACY_SALT: 'fixture' }, + outboundService: async request => { + const body = await request.json(); + if (request.url.includes('openrouter.ai')) return WorkerResponse.json({ model: body.model, usage: { cost: 0.0001, prompt_tokens: 200, completion_tokens: 10, prompt_tokens_details: { cached_tokens: 50 } }, choices: [{ message: { content: 'A' } }] }); + const answers = Object.fromEntries(Object.entries(body.questions).map(([id, q]) => { + if (q.type === 'noul') return [id, { noul: 0.9 }]; + const keys = Object.keys(q.criteria); + return [id, { choice: keys[0], confidence: 0.51, probabilities: Object.fromEntries(keys.map((k, i) => [k, i ? 0.49 : 0.51])) }]; + })); + return WorkerResponse.json({ model: 'jev-1.13.0', ...(missing ? {} : { usage: { input_tokens: 100, output_tokens: 5 } }), answers }); + }, +}] })); +const send = (body, path = '/v1/classify', json = true) => mf.dispatchFetch(`https://classifier.dev${path}`, { + ...(body ? { method: 'POST', body: JSON.stringify(body) } : {}), + headers: { 'content-type': 'application/json', 'cf-connecting-ip': '203.0.113.70', ...(json ? { accept: 'application/json' } : {}) }, +}); +try { + await mf.ready; + for (const [name, body, path] of [ + ['batch', { inputs: ['Invoice', 'Refund'], labels: ['billing', 'support'] }], + ['dimensions', { items: ['Invoice'], dimensions: { team: ['billing', 'support'], priority: ['high', 'low'] } }], + ['multi-label', { inputs: ['Invoice'], labels: ['billing', 'support'], multi: true }], + ['GET JSON', null, '/billing,support/Invoice'], + ['smart', { inputs: ['Invoice'], labels: ['billing', 'support'], tier: 'smart' }], + ['unknown usage', { inputs: ['Invoice'], labels: ['billing', 'support'] }], + ]) { + missing = name === 'unknown usage'; + const response = await send(body, path); + assert.equal(response.status, 200, await response.clone().text()); + const data = await response.json(); + const smart = name === 'smart'; + assert.equal(data.usage.input_tokens, missing ? null : smart ? 300 : 100); + assert.equal(data.usage.output_tokens, missing ? null : smart ? 15 : 5); + assert.equal(data.usage.total_tokens, missing ? null : smart ? 315 : 105); + assert.equal(data.usage.models.length, smart ? 2 : 1); + assert.equal(data.pricing.currency, 'USD'); + assert.equal(data.pricing.total_usd, 0); + assert.equal(data.pricing.billing_status, 'not_billed'); + assert.equal(data.pricing.estimated_usd, missing ? null : smart ? 0.0020042 : 0.0000042); + assert.equal(data.pricing.input_usd_per_million, 0.042); + assert.equal(data.pricing.usd_per_escalation, 0.002); + report.results.push({ name, response: data }); + } + missing = false; + const plain = await send(null, '/billing,support/Invoice', false); + assert.equal(await plain.text(), 'billing\n'); + const invalid = await send({ inputs: [], labels: ['billing', 'support'] }); + assert.equal(invalid.status, 400); + report.results.push({ name: 'plain text unchanged and invalid input rejected', status: 'passed' }); +} finally { + await mkdir('captures', { recursive: true }); + await writeFile('captures/usage-e2e.json', JSON.stringify(report, null, 2)); + await mf.dispose(); +} +console.log(`${report.results.length} usage E2E scenarios passed; captures/usage-e2e.json`); diff --git a/package.json b/package.json index 0dbed74..eed2229 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "build:admin": "vite build --config vite.admin.config.ts", "prebuild": "npm run build:admin", "predev": "npm run build:admin", - "test:e2e:spending": "node e2e/spending.mjs" + "test:e2e:spending": "node e2e/spending.mjs", + "test:e2e:usage": "node e2e/usage.mjs" }, "keywords": [], "author": "Michael Ryaboy", diff --git a/src/classification-usage.ts b/src/classification-usage.ts new file mode 100644 index 0000000..2ec3ec2 --- /dev/null +++ b/src/classification-usage.ts @@ -0,0 +1,41 @@ +import type { Meter, TokenCounts } from "./cost"; +import { CLASSIFICATION_PRICING, classificationCharge, classificationInputTokens, INPUT_PRICE_PER_MILLION } from "./lib/classification-pricing"; + +export function classificationUsage(meter: Meter) { + const sum = (field: keyof TokenCounts): number | null => { + if (!meter.tokens.length || meter.tokens.some(row => row[field] === null)) return null; + const total = meter.tokens.reduce((n, row) => n + row[field]!, 0); + return Number.isSafeInteger(total) ? total : null; + }; + const input = sum("inputTokens"), output = sum("outputTokens"); + const total = input === null || output === null ? null : input + output; + return { + input_tokens: input, output_tokens: output, + total_tokens: total !== null && Number.isSafeInteger(total) ? total : null, + cached_input_tokens: sum("cachedInputTokens"), + models: meter.tokens.map(row => ({ provider: row.provider, model: row.model, calls: row.calls, + input_tokens: row.inputTokens, output_tokens: row.outputTokens, cached_input_tokens: row.cachedInputTokens })), + }; +} + +export function classificationPricing(meter: Meter, escalations: number) { + const input = classificationInputTokens(meter.tokens); + return { + currency: "USD", + rate_version: CLASSIFICATION_PRICING.version, + input_tokens: input, + escalations, + input_usd_per_million: INPUT_PRICE_PER_MILLION, + usd_per_escalation: CLASSIFICATION_PRICING.escalationNanodollars / 1e9, + estimated_usd: input === null ? null : Number(classificationCharge(input, escalations).nanodollars) / 1e9, + total_usd: 0, + billing_status: "not_billed", + }; +} + +export async function withResponsePricing(response: Response, pricing: object): Promise { + const body = await response.json() as Record; + const headers = new Headers(response.headers); + headers.delete("content-length"); + return Response.json({ ...body, pricing }, { status: response.status, headers }); +} diff --git a/src/http/classification.ts b/src/http/classification.ts index c5839cd..0b0f19d 100644 --- a/src/http/classification.ts +++ b/src/http/classification.ts @@ -1,3 +1,4 @@ +import { withResponsePricing } from "../classification-usage"; import { spendingClassification } from "./spending-classification"; import worker, { type Env } from "../index"; import { newMeter } from "../cost"; @@ -111,5 +112,19 @@ export async function accountClassification(request: Request, env: AppEnv & Part headers.set("cache-control", "no-store"); headers.set("x-request-id", reservation.id); headers.set("x-billing-status", response.ok ? (priceTokens(card, meter.tokens) ? "settled" : "review") : "refunded"); - return new Response(response.body, { status: response.status, headers }); + const resultResponse = new Response(response.body, { status: response.status, headers }); + const charge = priceTokens(card, meter.tokens); + return response.ok && !typeSafe + ? withResponsePricing(resultResponse, { currency: "USD", rate_version: card.version, + total_usd: charge ? Number(charge.nanodollars) / 1e9 : null, + billing_status: charge ? "settled" : "review", + models: meter.tokens.map(row => { + const rate = card.models.find(rate => rate.provider === row.provider && rate.model === row.model); + return { provider: row.provider, model: row.model, + input_usd_per_million: rate ? Number(rate.inputNanodollars) / 1000 : null, + output_usd_per_million: rate ? Number(rate.outputNanodollars) / 1000 : null, + cached_input_usd_per_million: rate ? Number(rate.cachedInputNanodollars) / 1000 : null }; + }), + }) + : resultResponse; } diff --git a/src/http/spending-classification.ts b/src/http/spending-classification.ts index 49a4e05..f4135b0 100644 --- a/src/http/spending-classification.ts +++ b/src/http/spending-classification.ts @@ -1,3 +1,4 @@ +import { classificationPricing, withResponsePricing } from "../classification-usage"; import worker, { type Env } from "../index"; import { newMeter } from "../cost"; import { AppError, hashToken, now, type AppEnv } from "../server/db"; @@ -104,7 +105,14 @@ export async function spendingClassification(request: Request, env: AppEnv & Par headers.set("x-usage-cost-usd", (Number(charge.nanodollars) / 1e9).toFixed(9)); } headers.set("x-request-id", id); headers.set("x-billing-status", "pending"); headers.set("cache-control", "no-store"); - return new Response(response.body, { status: response.status, headers }); + const resultResponse = new Response(response.body, { status: response.status, headers }); + return response.ok && new URL(request.url).pathname !== "/v1/systemone" + ? withResponsePricing(resultResponse, { + ...classificationPricing(meter, typeof escalations === "number" ? escalations : 0), + total_usd: charge ? Number(charge.nanodollars) / 1e9 : null, + billing_status: "pending", + }) + : resultResponse; } catch (error) { if (error instanceof SpendingError) return errorResponse(error); throw error; diff --git a/src/index.ts b/src/index.ts index d13987f..97fd81d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,4 @@ +import { classificationUsage, classificationPricing } from "./classification-usage"; import { withFreeSpending, boundedRequest, freePreflight, type SpendingEnv } from "./spending"; import { SpendingError, errorResponse } from "./spending/policy"; import { providerFetch } from "./spending/permit"; @@ -2356,11 +2357,14 @@ const worker = { .map(([name, duration]) => `${name};dur=${duration.toFixed(2)}`).join(", "); } + const tokenUsage = classificationUsage(meter); + const pricing = classificationPricing(meter, results.filter(r => r.escalated).length); if (dimensions && matrix) { return json({ + pricing, tier, ...modelSummary, results: matrix.map((row) => ({ dimensions: Object.fromEntries(dimensions.map((d, i) => [d.name, row[i]])) })), - usage: { items: inputs.length, dimensions: dimensions.length, classifications: results.length, + usage: { ...tokenUsage, items: inputs.length, dimensions: dimensions.length, classifications: results.length, escalated: results.filter((r) => r.escalated).length, ...(escalationFailed ? { escalation_failed: escalationFailed } : {}), fallback: fallbackDecisions, ms }, @@ -2375,7 +2379,7 @@ const worker = { return text(body + "\n", 200, headers); } if (req.method === "GET") { - return json({ ...results[0], tier }, 200, headers); + return json({ ...results[0], tier, usage: tokenUsage, pricing }, 200, headers); } return json( { @@ -2394,7 +2398,9 @@ const worker = { model: r.model, }, ), + pricing, usage: { + ...tokenUsage, classifications: results.length, escalated: results.filter((r) => r.escalated).length, ...(escalationFailed ? { escalation_failed: escalationFailed } : {}), diff --git a/src/openapi.ts b/src/openapi.ts index 31d7c4e..c75ae82 100644 --- a/src/openapi.ts +++ b/src/openapi.ts @@ -1008,9 +1008,40 @@ export const OPENAPI = { max_labels: { type: "integer", minimum: 1, description: "Cap on how many multi-label answers come back; implies multi. A numeric string is read; zero or less means no cap; a fraction is rounded down." }, }, }, + TokenUsage: { + type: "object", + properties: { + input_tokens: { type: ["integer", "null"], description: "Provider-reported input tokens across all answered calls, including smart escalation. Null if any count is unavailable." }, + output_tokens: { type: ["integer", "null"] }, + total_tokens: { type: ["integer", "null"], description: "Input plus output; null if either is unknown." }, + cached_input_tokens: { type: ["integer", "null"], description: "Subset of input tokens, never added again to total_tokens." }, + models: { type: "array", items: { type: "object", properties: { + provider: { type: "string" }, model: { type: "string" }, calls: { type: "integer" }, + input_tokens: { type: ["integer", "null"] }, output_tokens: { type: ["integer", "null"] }, cached_input_tokens: { type: ["integer", "null"] }, + } } }, + }, + }, + ClassificationPricing: { + type: "object", + description: "USD customer pricing, separate from upstream provider spend. total_usd is zero for unbilled requests, null when unknown, and before credit rounding. Pending settlement is not a receipt. Token totals include recovery and escalation; pricing.input_tokens includes only billable primary calls. Plain-text responses and TypeSafe-compatible responses retain their existing shapes.", + properties: { + currency: { const: "USD" }, rate_version: { type: "string" }, + billing_status: { enum: ["not_billed", "pending", "settled", "review"] }, + total_usd: { type: ["number", "null"] }, + estimated_usd: { type: ["number", "null"], description: "Published classification price before free access or settlement; not an additional charge." }, + input_tokens: { type: ["integer", "null"] }, escalations: { type: "integer" }, + input_usd_per_million: { type: "number" }, usd_per_escalation: { type: "number" }, + models: { type: "array", description: "Legacy token-billed workspaces: the applied rate per provider/model, instead of the input-plus-escalation tariff.", items: { type: "object", properties: { + provider: { type: "string" }, model: { type: "string" }, + input_usd_per_million: { type: ["number", "null"] }, output_usd_per_million: { type: ["number", "null"] }, cached_input_usd_per_million: { type: ["number", "null"] }, + } } }, + }, + }, SingleResult: { type: "object", properties: { + usage: { $ref: "#/components/schemas/TokenUsage" }, + pricing: { $ref: "#/components/schemas/ClassificationPricing" }, label: { type: "string" }, confidence: { type: ["number", "null"], @@ -1054,6 +1085,7 @@ export const OPENAPI = { ClassifyResponse: { type: "object", properties: { + pricing: { $ref: "#/components/schemas/ClassificationPricing" }, tier: { type: "string" }, model: { type: "string", @@ -1087,6 +1119,7 @@ export const OPENAPI = { }, usage: { type: "object", + allOf: [{ $ref: "#/components/schemas/TokenUsage" }], properties: { items: { type: "integer", description: "Dimensions mode: number of input items." }, dimensions: { type: "integer", description: "Dimensions mode: number of fields per item." }, diff --git a/tests/spending.e2e.test.ts b/tests/spending.e2e.test.ts index b6c854f..9064e90 100644 --- a/tests/spending.e2e.test.ts +++ b/tests/spending.e2e.test.ts @@ -303,6 +303,9 @@ test("funded HTTP classification skips Spur/free budget and bills the published const r = await accountClassification(request("203.0.113.1", undefined, undefined, { authorization: `Bearer ${enrolled.secret}` }), env, "API", s.ctx); expect(r?.status).toBe(200); expect(r?.headers.get("x-billing-status")).toBe("pending"); + const payload = await r!.clone().json() as { usage: { input_tokens: number }; pricing: Record }; + expect(payload.usage.input_tokens).toBe(100); + expect(payload.pricing).toMatchObject({ currency: "USD", billing_status: "pending", total_usd: 0.0000042, estimated_usd: 0.0000042 }); await s.flush(); const row = await env.APP_DB.prepare("SELECT status,actual_nano::text AS nano FROM app_usage WHERE id=?").bind(r!.headers.get("x-request-id")).first(); expect(row).toEqual({ status: "completed", nano: "4200" });