Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions packages/opencode/src/session/retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ export const RETRY_BACKOFF_FACTOR = 2
export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout

const RETRYABLE_MESSAGE_PATTERNS = [
/429|500|502|503|504|524/i,
/rate increased too quickly|rate limit|rate-limit|rate_limit|too many requests/i,
/overloaded|service unavailable|service_unavailable|service-unavailable|internal error|internal_error|internal server error|server error|server_error|server-error|provider returned error|provider_returned_error|provider-returned-error/i,
/terminated|fetch failed|failed to fetch|network error|upstream connect|connection error|connection refused|connection lost|socket connection was closed|socket hang up|reset before headers|getaddrinfo|enotfound|eai_again|econnrefused|econnreset|etimedout/i,
/^timeout$|\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i,
/try your request again|retry your request|resource exhausted|resource_exhausted/i,
]

function cap(ms: number) {
return Math.min(ms, RETRY_MAX_DELAY)
}
Expand Down Expand Up @@ -72,7 +81,12 @@ export function retryable(error: Err, provider: string) {
const status = error.data.statusCode
// 5xx errors are transient server failures and should always be retried,
// even when the provider SDK doesn't explicitly mark them as retryable.
if (!error.data.isRetryable && !(status !== undefined && status >= 500)) return undefined
if (
!error.data.isRetryable &&
!(status !== undefined && status >= 500) &&
!matchesRetryableMessage(error.data.message) &&
!matchesRetryableMessage(error.data.responseBody)
) return undefined
if (error.data.responseBody?.includes("FreeUsageLimitError")) {
return {
message: GO_UPSELL_MESSAGE,
Expand Down Expand Up @@ -125,19 +139,16 @@ export function retryable(error: Err, provider: string) {
const message = isRecord(error.data) ? error.data.message : undefined
if (typeof message !== "string") return undefined
const lower = message.toLowerCase()
if (
lower.includes("rate increased too quickly") ||
lower.includes("rate limit") ||
lower.includes("rate_limit") ||
lower.includes("too many requests")
) {
return { message }
}
if (lower.includes("too_many_requests")) return { message: "Too Many Requests" }
if (lower.includes("exhausted") || lower.includes("unavailable")) return { message: "Provider is overloaded" }
if (matchesRetryableMessage(message)) return { message }
return undefined
}

function matchesRetryableMessage(value: unknown) {
return typeof value === "string" && RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(value))
}

function str(value: unknown) {
if (value === undefined || value === null) return ""
return String(value)
Expand Down
39 changes: 39 additions & 0 deletions packages/opencode/test/session/retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,45 @@ describe("session.retry.retryable", () => {
expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: msg })
})

test.each([
"Internal server error",
"internal error",
"server-error",
"Provider returned error",
"provider-returned-error",
"terminated",
"fetch failed",
"connection refused",
"connect ECONNREFUSED",
"request ETIMEDOUT",
"failed to fetch",
"EAI_AGAIN",
"response timed out",
"Please retry your request",
"try your request again",
"upstream returned status 524",
])("retries matching API error text: %s", (message) => {
expect(SessionRetry.retryable(wrap(message), retryProvider)).toEqual({ message })
})

test("retries hyphenated service-unavailable errors", () => {
expect(SessionRetry.retryable(wrap("service-unavailable"), retryProvider)).toEqual({
message: "Provider is overloaded",
})
})

test("matches retryable API response bodies", () => {
const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)(
new SessionV1.APIError({
message: "Request failed",
isRetryable: false,
statusCode: 400,
responseBody: JSON.stringify({ error: { message: "upstream connection refused" } }),
}).toObject(),
)
expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Request failed" })
})

test("retries transport timeout errors", () => {
const request = MessageV2.fromError(new ProviderError.HeaderTimeoutError(10000), { providerID })
expect(SessionV1.APIError.isInstance(request)).toBe(true)
Expand Down
Loading