diff --git a/.changeset/openrouter-combined-tools-and-schema.md b/.changeset/openrouter-combined-tools-and-schema.md new file mode 100644 index 0000000000..953df9f5fc --- /dev/null +++ b/.changeset/openrouter-combined-tools-and-schema.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai-openrouter': minor +--- + +Add native combined tools + `outputSchema` mode to both OpenRouter text adapters (chat-completions and Responses). When the resolved upstream model supports emitting a schema-constrained final answer alongside tool calls in a single pass, `chat({ outputSchema, tools, stream: true })` now wires the JSON Schema into the same streaming request as the tools and harvests the final-turn JSON, skipping the separate finalization round-trip. + +Because OpenRouter is a routing layer, capability is keyed per resolved upstream model via the new `OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS` set, exported from `@tanstack/ai-openrouter/model-meta`, which both adapters consult from `supportsCombinedToolsAndSchema()`. The set is derived from each upstream provider's combined-mode gate (Anthropic 4.5+, Gemini 3.x, OpenAI's strict `json_schema` era, Grok 4.x) rather than the broader catalog `responseFormat` flag, so models that advertise structured output but predate native combined mode stay on the legacy finalization path. diff --git a/docs/adapters/openrouter.md b/docs/adapters/openrouter.md index 3060bff5f4..91ffdb9dc8 100644 --- a/docs/adapters/openrouter.md +++ b/docs/adapters/openrouter.md @@ -112,6 +112,135 @@ export async function POST(request: Request) { } ``` +## Tools and structured output together + +You can pass both `tools` and `outputSchema` on one `chat()` call. For some +upstream models OpenRouter can return the typed object in that same streaming +request, so the engine does not make a second finalization call. + +That happens only when **every** model that can receive the request is in +`OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS`. The set is the upstream models +that already support this combined mode: + +- Anthropic Claude 4.5 and later +- Gemini 3.x text models +- Grok 4.x (not the multi-agent variant) +- OpenAI models from the `gpt-4o-2024-08-06` strict JSON Schema era onward + (the unpinned `openai/gpt-4o-mini` alias is included; the dated pin + `openai/gpt-4o-mini-2024-07-18` is not) + +If any fallback in `modelOptions.models` is outside that set, OpenRouter keeps +the two-call path. Routing suffixes such as `:nitro` do not change the gate. + +Import the set from `@tanstack/ai-openrouter/model-meta` if you need to check a +model before you send: + +```typescript +import { OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS } from "@tanstack/ai-openrouter/model-meta"; + +OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS.has("openai/gpt-5.5"); +``` + +Chat Completions (`openRouterText`) and Responses (`openRouterResponsesText`) +both attach the schema on this path. The client does not change: `useChat({ +outputSchema })` still reads `partial` and `final`. + +Server (Chat Completions): + +```typescript +import { chat, toServerSentEventsResponse, toolDefinition } from "@tanstack/ai"; +import { openRouterText } from "@tanstack/ai-openrouter"; +import { z } from "zod"; + +const getWeather = toolDefinition({ + name: "get_weather", + description: "Get the current weather", + inputSchema: z.object({ location: z.string() }), +}).server(async ({ location }) => { + return { temperature: 72, conditions: "sunny", location }; +}); + +const AnswerSchema = z.object({ + summary: z.string(), + location: z.string(), +}); + +export async function POST(request: Request) { + const { messages } = await request.json(); + + const stream = chat({ + adapter: openRouterText("openai/gpt-5.5"), + messages, + tools: [getWeather], + outputSchema: AnswerSchema, + stream: true, + }); + + return toServerSentEventsResponse(stream); +} +``` + +Server (Responses). Same `tools` and `outputSchema` as the Chat Completions +example, with `openRouterResponsesText`: + +```typescript +import { chat, toServerSentEventsResponse, toolDefinition } from "@tanstack/ai"; +import { openRouterResponsesText } from "@tanstack/ai-openrouter"; +import { z } from "zod"; + +const getWeather = toolDefinition({ + name: "get_weather", + description: "Get the current weather", + inputSchema: z.object({ location: z.string() }), +}).server(async ({ location }) => { + return { temperature: 72, conditions: "sunny", location }; +}); + +const AnswerSchema = z.object({ + summary: z.string(), + location: z.string(), +}); + +export async function POST(request: Request) { + const { messages } = await request.json(); + + const stream = chat({ + adapter: openRouterResponsesText("openai/gpt-5.5"), + messages, + tools: [getWeather], + outputSchema: AnswerSchema, + stream: true, + }); + + return toServerSentEventsResponse(stream); +} +``` + +Client: + +```tsx +import { useChat, fetchServerSentEvents } from "@tanstack/ai-react"; +import { z } from "zod"; + +const AnswerSchema = z.object({ + summary: z.string(), + location: z.string(), +}); + +const { sendMessage, partial, final } = useChat({ + connection: fetchServerSentEvents("/api/chat"), + outputSchema: AnswerSchema, +}); +``` + +See [Structured Outputs with tools](../structured-outputs/with-tools) for the +event order, and [Middleware](../advanced/middleware) for how +`structuredOutput` phase behaves on this path. + +To try this in a browser, run `examples/ts-react-chat` and open +`/generations/openrouter-combined`. The page shows the tool call, the typed +object, and the adapter call counts. `structuredOutputStream` must stay at 0. + ## Environment Variables Set your API key in environment variables: diff --git a/docs/advanced/middleware.md b/docs/advanced/middleware.md index 80fe5fed46..334bb3ead2 100644 --- a/docs/advanced/middleware.md +++ b/docs/advanced/middleware.md @@ -102,7 +102,7 @@ The context's `phase` field tracks where you are in the lifecycle: | `modelStream` | While adapter streams chunks | `onChunk`, `onUsage` | | `beforeTools` | Before tool execution | `onBeforeToolCall` | | `afterTools` | After tool execution | `onAfterToolCall` | -| `structuredOutput` | During the final structured-output adapter call (when `outputSchema` is set **and** the adapter does not declare `supportsCombinedToolsAndSchema()`). Chunks from `adapter.structuredOutputStream` (or the synthesized non-streaming fallback) flow through `onChunk` with this phase, and `onUsage` fires for the final call's tokens. **Does not fire** for adapters that natively combine tools + schema in one streaming call (modern OpenAI Chat Completions, OpenAI Responses, Claude 4.5+, Gemini 3.x, Grok 4.x family — see issue #605); on that path middleware observes the run through `beforeModel` / `modelStream` as usual. | `onStructuredOutputConfig`, `onConfig`, `onChunk`, `onUsage` | +| `structuredOutput` | During the final structured-output adapter call (when `outputSchema` is set **and** the adapter does not declare `supportsCombinedToolsAndSchema()`). Chunks from `adapter.structuredOutputStream` (or the synthesized non-streaming fallback) flow through `onChunk` with this phase, and `onUsage` fires for the final call's tokens. **Does not fire** for adapters that natively combine tools + schema in one streaming call (modern OpenAI Chat Completions, OpenAI Responses, Claude 4.5+, Gemini 3.x, Grok 4.x family, and OpenRouter when every routed model is in `OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS`; see issue #605). On that path middleware observes the run through `beforeModel` / `modelStream` as usual. | `onStructuredOutputConfig`, `onConfig`, `onChunk`, `onUsage` | ## Hooks Reference diff --git a/docs/config.json b/docs/config.json index 6ab97268ed..ca435a0054 100644 --- a/docs/config.json +++ b/docs/config.json @@ -337,7 +337,7 @@ "label": "Overview", "to": "structured-outputs/overview", "addedAt": "2026-05-19", - "updatedAt": "2026-08-18" + "updatedAt": "2026-08-19" }, { "label": "One-Shot Extraction", @@ -360,7 +360,7 @@ "label": "With Tools", "to": "structured-outputs/with-tools", "addedAt": "2026-05-19", - "updatedAt": "2026-08-14" + "updatedAt": "2026-08-19" }, { "label": "Harness Agents", @@ -490,7 +490,7 @@ "label": "Middleware", "to": "advanced/middleware", "addedAt": "2026-04-15", - "updatedAt": "2026-07-21" + "updatedAt": "2026-08-19" }, { "label": "Built-in Middleware", @@ -857,7 +857,7 @@ "label": "OpenRouter Adapter", "to": "adapters/openrouter", "addedAt": "2026-04-15", - "updatedAt": "2026-08-13" + "updatedAt": "2026-08-19" }, { "label": "Perplexity Search", diff --git a/docs/structured-outputs/overview.md b/docs/structured-outputs/overview.md index 79d1ad1c3f..dba56fe729 100644 --- a/docs/structured-outputs/overview.md +++ b/docs/structured-outputs/overview.md @@ -113,10 +113,14 @@ exactly once at the end of the entire run. > - Anthropic Claude 4.5+ > - Gemini 3.x > - Grok 4.x family +> - OpenRouter, when every routed model is in +> `OPENROUTER_COMBINED_TOOLS_AND_SCHEMA_MODELS` (see the +> [OpenRouter adapter](../adapters/openrouter.md#tools-and-structured-output-together)) > > **Adapters without native combined-mode support** (Anthropic 4.4-, Gemini -> 2.x, Grok 2/3, Groq, Ollama, OpenRouter) keep the legacy finalization -> path and the `'structuredOutput'` phase fires as before. +> 2.x, Grok 2/3, Groq, Ollama, and OpenRouter models outside that set) keep +> the legacy finalization path and the `'structuredOutput'` phase fires as +> before. ### Observing structured-output chunks diff --git a/docs/structured-outputs/with-tools.md b/docs/structured-outputs/with-tools.md index 7ac8aec506..8e64503f52 100644 --- a/docs/structured-outputs/with-tools.md +++ b/docs/structured-outputs/with-tools.md @@ -17,6 +17,10 @@ You want the agent to use tools to gather information, then return a structured This page covers the combined `outputSchema` + `tools` shape, including the pause/resume points (server-tool approval prompts, client-tool invocations) that can land mid-run before the structured object arrives. +On adapters that support native combined mode (modern OpenAI, Claude 4.5+, Gemini 3.x, Grok 4.x, and [OpenRouter on those same upstream models](../adapters/openrouter.md#tools-and-structured-output-together)), `chat({ tools, outputSchema, stream: true })` is one streaming request. The extra finalization call does not run. + +The React chat example has a live OpenRouter page at `/generations/openrouter-combined`. + > **Note:** If you're not yet familiar with how tools work in TanStack AI, read [Tool Architecture](../tools/tool-architecture) and [Server Tools](../tools/server-tools) first. The patterns here build on the regular agent-loop flow — `outputSchema` just adds a final terminal event. ## Non-streaming: tools first, then structured object diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index 8660905a3c..aba382f7a3 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -204,6 +204,19 @@ export default function Header() { Structured Output + setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-1" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-1', + }} + > + + OpenRouter Combined + + setIsOpen(false)} diff --git a/examples/ts-react-chat/src/routeTree.gen.ts b/examples/ts-react-chat/src/routeTree.gen.ts index e42124b47d..bc690e9d93 100644 --- a/examples/ts-react-chat/src/routeTree.gen.ts +++ b/examples/ts-react-chat/src/routeTree.gen.ts @@ -35,6 +35,7 @@ import { Route as GenerationsStructuredOutputRouteImport } from './routes/genera import { Route as GenerationsStructuredChatRouteImport } from './routes/generations.structured-chat' import { Route as GenerationsSpeechRouteImport } from './routes/generations.speech' import { Route as GenerationsPersistentGenerationRouteImport } from './routes/generations.persistent-generation' +import { Route as GenerationsOpenrouterCombinedRouteImport } from './routes/generations.openrouter-combined' import { Route as GenerationsImageRouteImport } from './routes/generations.image' import { Route as GenerationsAudioRouteImport } from './routes/generations.audio' import { Route as ExampleRuntimeContextRouteImport } from './routes/example.runtime-context' @@ -48,6 +49,7 @@ import { Route as ApiSandboxTriageRouteImport } from './routes/api.sandbox-triag import { Route as ApiSandboxRepoReportRouteImport } from './routes/api.sandbox-repo-report' import { Route as ApiResumableRouteImport } from './routes/api.resumable' import { Route as ApiPersistentChatRouteImport } from './routes/api.persistent-chat' +import { Route as ApiOpenrouterCombinedRouteImport } from './routes/api.openrouter-combined' import { Route as ApiMcpStatusRouteImport } from './routes/api.mcp-status' import { Route as ApiMcpPoolRouteImport } from './routes/api.mcp-pool' import { Route as ApiMcpManualRouteImport } from './routes/api.mcp-manual' @@ -203,6 +205,12 @@ const GenerationsPersistentGenerationRoute = path: '/generations/persistent-generation', getParentRoute: () => rootRouteImport, } as any) +const GenerationsOpenrouterCombinedRoute = + GenerationsOpenrouterCombinedRouteImport.update({ + id: '/generations/openrouter-combined', + path: '/generations/openrouter-combined', + getParentRoute: () => rootRouteImport, + } as any) const GenerationsImageRoute = GenerationsImageRouteImport.update({ id: '/generations/image', path: '/generations/image', @@ -268,6 +276,11 @@ const ApiPersistentChatRoute = ApiPersistentChatRouteImport.update({ path: '/api/persistent-chat', getParentRoute: () => rootRouteImport, } as any) +const ApiOpenrouterCombinedRoute = ApiOpenrouterCombinedRouteImport.update({ + id: '/api/openrouter-combined', + path: '/api/openrouter-combined', + getParentRoute: () => rootRouteImport, +} as any) const ApiMcpStatusRoute = ApiMcpStatusRouteImport.update({ id: '/api/mcp-status', path: '/api/mcp-status', @@ -403,6 +416,7 @@ export interface FileRoutesByFullPath { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/openrouter-combined': typeof ApiOpenrouterCombinedRoute '/api/persistent-chat': typeof ApiPersistentChatRoute '/api/resumable': typeof ApiResumableRoute '/api/sandbox-repo-report': typeof ApiSandboxRepoReportRoute @@ -416,6 +430,7 @@ export interface FileRoutesByFullPath { '/example/runtime-context': typeof ExampleRuntimeContextRoute '/generations/audio': typeof GenerationsAudioRoute '/generations/image': typeof GenerationsImageRoute + '/generations/openrouter-combined': typeof GenerationsOpenrouterCombinedRoute '/generations/persistent-generation': typeof GenerationsPersistentGenerationRoute '/generations/speech': typeof GenerationsSpeechRoute '/generations/structured-chat': typeof GenerationsStructuredChatRoute @@ -464,6 +479,7 @@ export interface FileRoutesByTo { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/openrouter-combined': typeof ApiOpenrouterCombinedRoute '/api/persistent-chat': typeof ApiPersistentChatRoute '/api/resumable': typeof ApiResumableRoute '/api/sandbox-repo-report': typeof ApiSandboxRepoReportRoute @@ -477,6 +493,7 @@ export interface FileRoutesByTo { '/example/runtime-context': typeof ExampleRuntimeContextRoute '/generations/audio': typeof GenerationsAudioRoute '/generations/image': typeof GenerationsImageRoute + '/generations/openrouter-combined': typeof GenerationsOpenrouterCombinedRoute '/generations/persistent-generation': typeof GenerationsPersistentGenerationRoute '/generations/speech': typeof GenerationsSpeechRoute '/generations/structured-chat': typeof GenerationsStructuredChatRoute @@ -526,6 +543,7 @@ export interface FileRoutesById { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/openrouter-combined': typeof ApiOpenrouterCombinedRoute '/api/persistent-chat': typeof ApiPersistentChatRoute '/api/resumable': typeof ApiResumableRoute '/api/sandbox-repo-report': typeof ApiSandboxRepoReportRoute @@ -539,6 +557,7 @@ export interface FileRoutesById { '/example/runtime-context': typeof ExampleRuntimeContextRoute '/generations/audio': typeof GenerationsAudioRoute '/generations/image': typeof GenerationsImageRoute + '/generations/openrouter-combined': typeof GenerationsOpenrouterCombinedRoute '/generations/persistent-generation': typeof GenerationsPersistentGenerationRoute '/generations/speech': typeof GenerationsSpeechRoute '/generations/structured-chat': typeof GenerationsStructuredChatRoute @@ -589,6 +608,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/openrouter-combined' | '/api/persistent-chat' | '/api/resumable' | '/api/sandbox-repo-report' @@ -602,6 +622,7 @@ export interface FileRouteTypes { | '/example/runtime-context' | '/generations/audio' | '/generations/image' + | '/generations/openrouter-combined' | '/generations/persistent-generation' | '/generations/speech' | '/generations/structured-chat' @@ -650,6 +671,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/openrouter-combined' | '/api/persistent-chat' | '/api/resumable' | '/api/sandbox-repo-report' @@ -663,6 +685,7 @@ export interface FileRouteTypes { | '/example/runtime-context' | '/generations/audio' | '/generations/image' + | '/generations/openrouter-combined' | '/generations/persistent-generation' | '/generations/speech' | '/generations/structured-chat' @@ -711,6 +734,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/openrouter-combined' | '/api/persistent-chat' | '/api/resumable' | '/api/sandbox-repo-report' @@ -724,6 +748,7 @@ export interface FileRouteTypes { | '/example/runtime-context' | '/generations/audio' | '/generations/image' + | '/generations/openrouter-combined' | '/generations/persistent-generation' | '/generations/speech' | '/generations/structured-chat' @@ -773,6 +798,7 @@ export interface RootRouteChildren { ApiMcpManualRoute: typeof ApiMcpManualRoute ApiMcpPoolRoute: typeof ApiMcpPoolRoute ApiMcpStatusRoute: typeof ApiMcpStatusRoute + ApiOpenrouterCombinedRoute: typeof ApiOpenrouterCombinedRoute ApiPersistentChatRoute: typeof ApiPersistentChatRoute ApiResumableRoute: typeof ApiResumableRoute ApiSandboxRepoReportRoute: typeof ApiSandboxRepoReportRoute @@ -786,6 +812,7 @@ export interface RootRouteChildren { ExampleRuntimeContextRoute: typeof ExampleRuntimeContextRoute GenerationsAudioRoute: typeof GenerationsAudioRoute GenerationsImageRoute: typeof GenerationsImageRoute + GenerationsOpenrouterCombinedRoute: typeof GenerationsOpenrouterCombinedRoute GenerationsPersistentGenerationRoute: typeof GenerationsPersistentGenerationRoute GenerationsSpeechRoute: typeof GenerationsSpeechRoute GenerationsStructuredChatRoute: typeof GenerationsStructuredChatRoute @@ -985,6 +1012,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof GenerationsPersistentGenerationRouteImport parentRoute: typeof rootRouteImport } + '/generations/openrouter-combined': { + id: '/generations/openrouter-combined' + path: '/generations/openrouter-combined' + fullPath: '/generations/openrouter-combined' + preLoaderRoute: typeof GenerationsOpenrouterCombinedRouteImport + parentRoute: typeof rootRouteImport + } '/generations/image': { id: '/generations/image' path: '/generations/image' @@ -1076,6 +1110,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiPersistentChatRouteImport parentRoute: typeof rootRouteImport } + '/api/openrouter-combined': { + id: '/api/openrouter-combined' + path: '/api/openrouter-combined' + fullPath: '/api/openrouter-combined' + preLoaderRoute: typeof ApiOpenrouterCombinedRouteImport + parentRoute: typeof rootRouteImport + } '/api/mcp-status': { id: '/api/mcp-status' path: '/api/mcp-status' @@ -1263,6 +1304,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMcpManualRoute: ApiMcpManualRoute, ApiMcpPoolRoute: ApiMcpPoolRoute, ApiMcpStatusRoute: ApiMcpStatusRoute, + ApiOpenrouterCombinedRoute: ApiOpenrouterCombinedRoute, ApiPersistentChatRoute: ApiPersistentChatRoute, ApiResumableRoute: ApiResumableRoute, ApiSandboxRepoReportRoute: ApiSandboxRepoReportRoute, @@ -1276,6 +1318,7 @@ const rootRouteChildren: RootRouteChildren = { ExampleRuntimeContextRoute: ExampleRuntimeContextRoute, GenerationsAudioRoute: GenerationsAudioRoute, GenerationsImageRoute: GenerationsImageRoute, + GenerationsOpenrouterCombinedRoute: GenerationsOpenrouterCombinedRoute, GenerationsPersistentGenerationRoute: GenerationsPersistentGenerationRoute, GenerationsSpeechRoute: GenerationsSpeechRoute, GenerationsStructuredChatRoute: GenerationsStructuredChatRoute, diff --git a/examples/ts-react-chat/src/routes/api.openrouter-combined.ts b/examples/ts-react-chat/src/routes/api.openrouter-combined.ts new file mode 100644 index 0000000000..1cce943826 --- /dev/null +++ b/examples/ts-react-chat/src/routes/api.openrouter-combined.ts @@ -0,0 +1,276 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequestBody, + EventType, + toServerSentEventsResponse, + toolDefinition, +} from '@tanstack/ai' +import { + openRouterResponsesText, + openRouterText, +} from '@tanstack/ai-openrouter' +import { z } from 'zod' +import type { ChatMiddleware, StreamChunk } from '@tanstack/ai' + +/** + * Live confirmation page for OpenRouter native combined mode: + * `chat({ tools, outputSchema, stream: true })` must stay on `chatStream` + * and must not call `structuredOutputStream`. + */ + +export const CitySchema = z.object({ + city: z.string().describe('City name'), + code: z.string().describe('Short 3-letter city code from the tool'), + summary: z.string().describe('One sentence that uses the city and the code'), +}) + +export type CityResult = z.infer + +const lookupCityCode = toolDefinition({ + name: 'lookup_city_code', + description: 'Return a short 3-letter code for a city', + inputSchema: z.object({ city: z.string() }), + outputSchema: z.object({ code: z.string() }), +}).server(async ({ city }) => ({ + code: city.slice(0, 3).toUpperCase(), +})) + +const PROVIDERS = ['openrouter', 'openrouter-responses'] as const +type Provider = (typeof PROVIDERS)[number] + +const MODELS = [ + 'openai/gpt-5.5', + 'anthropic/claude-sonnet-4.6', + 'google/gemini-3.5-flash', + 'x-ai/grok-4.5', +] as const +export type CombinedModel = (typeof MODELS)[number] + +export const COMBINED_MODELS: ReadonlyArray<{ + value: CombinedModel + label: string +}> = [ + { value: 'openai/gpt-5.5', label: 'OpenAI GPT-5.5' }, + { value: 'anthropic/claude-sonnet-4.6', label: 'Claude Sonnet 4.6' }, + { value: 'google/gemini-3.5-flash', label: 'Gemini 3.5 Flash' }, + { value: 'x-ai/grok-4.5', label: 'Grok 4.5' }, +] + +function isProvider(value: unknown): value is Provider { + return value === 'openrouter' || value === 'openrouter-responses' +} + +function isModel(value: unknown): value is CombinedModel { + return typeof value === 'string' && MODELS.some((model) => model === value) +} + +export type CombinedModeStats = { + supportsCombined: boolean + chatStreamCalls: number + chatStreamWithSchema: number + structuredOutputStreamCalls: number + structuredOutputCalls: number + nativeCombined: boolean +} + +/** + * Count adapter entry points so the UI can prove native combined mode: + * schema rides on `chatStream`, and `structuredOutputStream` stays at 0. + * + * Infer T from one concrete adapter (Chat Completions or Responses). A union + * of those two classes made the wrapper `options` parameter implicit `any`. + */ +function instrumentAdapter< + T extends { + model: string + chatStream: (options: never) => AsyncIterable + structuredOutputStream?: (options: never) => AsyncIterable + structuredOutput: (options: never) => Promise + supportsCombinedToolsAndSchema: () => boolean + }, +>(adapter: T): { adapter: T; snapshot: () => CombinedModeStats } { + const stats = { + supportsCombined: adapter.supportsCombinedToolsAndSchema() === true, + chatStreamCalls: 0, + chatStreamWithSchema: 0, + structuredOutputStreamCalls: 0, + structuredOutputCalls: 0, + } + + const origChatStream = adapter.chatStream.bind(adapter) + adapter.chatStream = (options: { outputSchema?: unknown }) => { + stats.chatStreamCalls += 1 + if (options.outputSchema) { + stats.chatStreamWithSchema += 1 + } + return origChatStream(options as never) + } + + const origStructuredStream = adapter.structuredOutputStream?.bind(adapter) + if (origStructuredStream) { + adapter.structuredOutputStream = (options: never) => { + stats.structuredOutputStreamCalls += 1 + return origStructuredStream(options) + } + } + + const origStructured = adapter.structuredOutput.bind(adapter) + adapter.structuredOutput = (options: never) => { + stats.structuredOutputCalls += 1 + return origStructured(options) + } + + return { + adapter, + snapshot: () => ({ + ...stats, + nativeCombined: + stats.supportsCombined && + stats.structuredOutputStreamCalls === 0 && + stats.structuredOutputCalls === 0 && + stats.chatStreamWithSchema > 0, + }), + } +} + +function phaseCounterMiddleware(): { + middleware: ChatMiddleware + snapshot: () => Record +} { + const counts: Record = {} + return { + middleware: { + name: 'phase-counter', + onChunk(ctx) { + counts[ctx.phase] = (counts[ctx.phase] ?? 0) + 1 + }, + }, + snapshot: () => ({ ...counts }), + } +} + +async function* withTrailingDiagnostics( + stream: AsyncIterable, + combinedSnapshot: () => CombinedModeStats, + phaseSnapshot: () => Record, + model: string, +): AsyncIterable { + let yielded = false + for await (const chunk of stream) { + if ( + chunk.type === EventType.RUN_FINISHED || + chunk.type === EventType.RUN_ERROR + ) { + yielded = true + yield { + type: EventType.CUSTOM, + name: 'combined-mode', + value: combinedSnapshot(), + model, + timestamp: Date.now(), + } + yield { + type: EventType.CUSTOM, + name: 'phase-counts', + value: phaseSnapshot(), + model, + timestamp: Date.now(), + } + } + yield chunk + } + if (!yielded) { + yield { + type: EventType.CUSTOM, + name: 'combined-mode', + value: combinedSnapshot(), + model, + timestamp: Date.now(), + } + yield { + type: EventType.CUSTOM, + name: 'phase-counts', + value: phaseSnapshot(), + model, + timestamp: Date.now(), + } + } +} + +export const Route = createFileRoute('/api/openrouter-combined')({ + server: { + handlers: { + POST: async ({ request }) => { + if (request.signal.aborted) { + return new Response(null, { status: 499 }) + } + + const abortController = new AbortController() + const onAbort = () => abortController.abort() + request.signal.addEventListener('abort', onAbort, { once: true }) + if (request.signal.aborted) { + onAbort() + } + + let params: Awaited> + try { + params = await chatParamsFromRequestBody(await request.json()) + } catch (error) { + return new Response( + error instanceof Error ? error.message : 'Bad request', + { status: 400 }, + ) + } + + try { + const provider: Provider = isProvider(params.forwardedProps.provider) + ? params.forwardedProps.provider + : 'openrouter' + const model: CombinedModel = isModel(params.forwardedProps.model) + ? params.forwardedProps.model + : 'openai/gpt-5.5' + + const { adapter, snapshot: combinedSnapshot } = + provider === 'openrouter-responses' + ? instrumentAdapter(openRouterResponsesText(model)) + : instrumentAdapter(openRouterText(model)) + const counter = phaseCounterMiddleware() + + const streamIterable = chat({ + adapter, + messages: params.messages, + tools: [lookupCityCode], + outputSchema: CitySchema, + stream: true, + middleware: [counter.middleware], + threadId: params.threadId, + runId: params.runId, + abortController, + }) as AsyncIterable + + const withDiagnostics = withTrailingDiagnostics( + streamIterable, + combinedSnapshot, + counter.snapshot, + adapter.model, + ) + + return toServerSentEventsResponse(withDiagnostics, { + abortController, + }) + } catch (error: unknown) { + const message = + error instanceof Error ? error.message : 'An error occurred' + console.error('[api/openrouter-combined] Error:', error) + return new Response(JSON.stringify({ error: message }), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } finally { + request.signal.removeEventListener('abort', onAbort) + } + }, + }, + }, +}) diff --git a/examples/ts-react-chat/src/routes/generations.openrouter-combined.tsx b/examples/ts-react-chat/src/routes/generations.openrouter-combined.tsx new file mode 100644 index 0000000000..457c15fda3 --- /dev/null +++ b/examples/ts-react-chat/src/routes/generations.openrouter-combined.tsx @@ -0,0 +1,394 @@ +import { useId, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { EventType } from '@tanstack/ai' +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import { CitySchema, COMBINED_MODELS } from './api.openrouter-combined' +import type { + CityResult, + CombinedModeStats, + CombinedModel, +} from './api.openrouter-combined' +import type { StreamChunk } from '@tanstack/ai' + +const SAMPLE_PROMPT = + 'Call lookup_city_code for Paris, then return the structured object with city, code, and a one-sentence summary.' + +type Provider = 'openrouter' | 'openrouter-responses' + +function isProvider(value: string): value is Provider { + return value === 'openrouter' || value === 'openrouter-responses' +} + +function isCombinedModel(value: string): value is CombinedModel { + return COMBINED_MODELS.some((option) => option.value === value) +} + +function isPhaseCounts(value: unknown): value is Record { + if (typeof value !== 'object' || value === null) return false + return Object.values(value).every((item) => typeof item === 'number') +} + +function isCombinedStats(value: unknown): value is CombinedModeStats { + if (typeof value !== 'object' || value === null) return false + if ( + !('supportsCombined' in value) || + !('chatStreamCalls' in value) || + !('chatStreamWithSchema' in value) || + !('structuredOutputStreamCalls' in value) || + !('structuredOutputCalls' in value) || + !('nativeCombined' in value) + ) { + return false + } + return ( + typeof value.supportsCombined === 'boolean' && + typeof value.chatStreamCalls === 'number' && + typeof value.chatStreamWithSchema === 'number' && + typeof value.structuredOutputStreamCalls === 'number' && + typeof value.structuredOutputCalls === 'number' && + typeof value.nativeCombined === 'boolean' + ) +} + +function OpenRouterCombinedPage() { + const providerId = useId() + const modelId = useId() + const promptId = useId() + const statusId = useId() + const [prompt, setPrompt] = useState(SAMPLE_PROMPT) + const [provider, setProvider] = useState('openrouter') + const [model, setModel] = useState('openai/gpt-5.5') + const [error, setError] = useState(null) + const [phaseCounts, setPhaseCounts] = useState | null>( + null, + ) + const [combined, setCombined] = useState(null) + + const resetLocal = () => { + setError(null) + setPhaseCounts(null) + setCombined(null) + } + + const handleChunk = (chunk: StreamChunk) => { + if (chunk.type !== EventType.CUSTOM) return + + if (chunk.name === 'combined-mode' && isCombinedStats(chunk.value)) { + setCombined(chunk.value) + } else if (chunk.name === 'phase-counts' && isPhaseCounts(chunk.value)) { + setPhaseCounts(chunk.value) + } + } + + const chat = useChat({ + threadId: 'openrouter-combined:useChat', + outputSchema: CitySchema, + connection: fetchServerSentEvents('/api/openrouter-combined'), + forwardedProps: { provider, model }, + onChunk: handleChunk, + onError: (err) => { + setError(err.message) + }, + }) + + const toolCalls = chat.messages + .flatMap((message) => message.parts) + .filter((part) => part.type === 'tool-call') + + const result: CityResult | null = chat.final ?? null + const isLoading = chat.isLoading + const structuredOutputPhase = phaseCounts?.structuredOutput ?? 0 + const nativeCombined = + combined?.nativeCombined === true && structuredOutputPhase === 0 + + const handleGenerate = async () => { + if (!prompt.trim()) return + resetLocal() + chat.clear() + await chat.sendMessage(prompt.trim()) + } + + const handleAbort = () => { + chat.stop() + setError('Aborted') + } + + const handleReset = () => { + resetLocal() + chat.clear() + } + + return ( +
+
+

OpenRouter combined mode

+

+ This page calls chat() with + both tools and{' '} + outputSchema. Native combined + mode is working when the model calls{' '} + lookup_city_code, returns a + typed object, and{' '} + structuredOutputStream stays + at 0. +

+
+ +
+
+
+
+ + +
+
+ + +
+
+ +
+ +