diff --git a/packages/ai/src/route/transport/http.ts b/packages/ai/src/route/transport/http.ts index 5e330f61d5c4..f6f709ea7117 100644 --- a/packages/ai/src/route/transport/http.ts +++ b/packages/ai/src/route/transport/http.ts @@ -120,16 +120,17 @@ export const httpJson =
(input: HttpJsonInput): HttpJs const http = RequestExecutor.responseHttp(response) const remaining = Duration.subtract(total, Duration.millis((yield* Clock.currentTimeMillis) - started)) return { - frames: prepared.framing.frame( - RequestExecutor.responseStream(response).pipe( - Stream.timeoutOrElse({ - duration: timeoutDuration(request.http?.chunkTimeout), - orElse: () => Stream.fail(timeout("read", "Timed out waiting for response data", http)), - }), - Stream.interruptWhen( - Effect.sleep(remaining).pipe( - Effect.andThen(Effect.fail(timeout("read", "Timed out waiting for the response to complete", http))), - ), + // Bound the stall on framed progress, not raw bytes. Framing drops SSE + // comment keepalives (`: keepalive`) and other empty events, so counting + // bytes would let a provider hold a stalled generation warm forever. + frames: prepared.framing.frame(RequestExecutor.responseStream(response)).pipe( + Stream.timeoutOrElse({ + duration: timeoutDuration(request.http?.chunkTimeout), + orElse: () => Stream.fail(timeout("read", "Timed out waiting for response data", http)), + }), + Stream.interruptWhen( + Effect.sleep(remaining).pipe( + Effect.andThen(Effect.fail(timeout("read", "Timed out waiting for the response to complete", http))), ), ), ), diff --git a/packages/ai/test/http-timeout.test.ts b/packages/ai/test/http-timeout.test.ts index 6b45a8070079..88cf21f4a0f8 100644 --- a/packages/ai/test/http-timeout.test.ts +++ b/packages/ai/test/http-timeout.test.ts @@ -49,6 +49,31 @@ const stalledServer = Effect.gen(function* () { return { layer, stalled, resume: () => resume() } }) +// Sends one content chunk, then only what the test asks for. Lets a test prove +// that bytes framing drops, such as SSE comment keepalives, are not progress. +const heartbeatServer = Effect.gen(function* () { + const started = yield* Deferred.make