Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions packages/core/src/v1/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,15 @@ export const Info = Schema.Struct({
description:
"Thresholds for truncating tool output. When output exceeds either limit, the full text is written to the truncation directory and a preview is returned.",
}),
webfetch: Schema.optional(
Schema.Struct({
max_response_size: Schema.optional(PositiveInt).annotate({
description: "Maximum webfetch response size in bytes. Defaults to 5242880 (5MB).",
}),
}),
).annotate({
description: "Webfetch tool limits",
}),
compaction: Schema.optional(
Schema.Struct({
auto: Schema.optional(Schema.Boolean).annotate({
Expand Down
31 changes: 25 additions & 6 deletions packages/opencode/src/tool/webfetch.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,25 @@
import { Effect, Schema } from "effect"
import { Effect, Option, Schema } from "effect"
import { Config } from "@/config/config"
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
import { Parser } from "htmlparser2"
import * as Tool from "./tool"
import TurndownService from "turndown"
import DESCRIPTION from "./webfetch.txt"
import { isImageAttachment } from "@/util/media"

const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB
export const DEFAULT_MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB

function parsePositiveIntEnv(raw: string | undefined) {
if (raw == null || raw === "") return undefined
const parsed = Number.parseInt(raw, 10)
if (!Number.isInteger(parsed) || parsed <= 0 || String(parsed) !== raw.trim()) return undefined
return parsed
}

export function resolveMaxResponseSize(config?: { webfetch?: { max_response_size?: number } }) {
if (config?.webfetch?.max_response_size != null) return config.webfetch.max_response_size
return parsePositiveIntEnv(process.env["OPENCODE_WEBFETCH_MAX_SIZE"]) ?? DEFAULT_MAX_RESPONSE_SIZE
}
const DEFAULT_TIMEOUT = 30 * 1000 // 30 seconds
const MAX_TIMEOUT = 120 * 1000 // 2 minutes

Expand Down Expand Up @@ -47,6 +60,12 @@ export const WebFetchTool = Tool.define(
},
})

const configSvc = yield* Effect.serviceOption(Config.Service)
const cfg = Option.isNone(configSvc)
? undefined
: yield* configSvc.value.get().pipe(Effect.catch(() => Effect.succeed(undefined)))
const maxResponseSize = resolveMaxResponseSize(cfg)

const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT)

// Build Accept header based on requested format with q parameters for fallbacks
Expand Down Expand Up @@ -94,13 +113,13 @@ export const WebFetchTool = Tool.define(

// Check content length
const contentLength = response.headers["content-length"]
if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
throw new Error("Response too large (exceeds 5MB limit)")
if (contentLength && parseInt(contentLength) > maxResponseSize) {
throw new Error("Response too large (exceeds configured size limit)")
}

const arrayBuffer = yield* response.arrayBuffer
if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
throw new Error("Response too large (exceeds 5MB limit)")
if (arrayBuffer.byteLength > maxResponseSize) {
throw new Error("Response too large (exceeds configured size limit)")
}

const contentType = response.headers["content-type"] || ""
Expand Down
56 changes: 54 additions & 2 deletions packages/opencode/test/tool/webfetch.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { describe, expect } from "bun:test"
import { describe, expect, test } from "bun:test"
import { LayerNode } from "@opencode-ai/core/effect/layer-node"
import { httpClient } from "@opencode-ai/core/effect/app-node-platform"
import { Effect, Layer } from "effect"
import { FetchHttpClient, HttpClient } from "effect/unstable/http"
import { Agent } from "../../src/agent/agent"
import { Truncate } from "@/tool/truncate"
import { WebFetchTool } from "../../src/tool/webfetch"
import { DEFAULT_MAX_RESPONSE_SIZE, resolveMaxResponseSize, WebFetchTool } from "../../src/tool/webfetch"
import { SessionID, MessageID } from "../../src/session/schema"
import { Tool } from "@/tool/tool"
import { testEffect } from "../lib/effect"
Expand Down Expand Up @@ -117,3 +117,55 @@ describe("tool.webfetch", () => {
),
)
})

describe("resolveMaxResponseSize", () => {
const envKey = "OPENCODE_WEBFETCH_MAX_SIZE"

test("defaults to 5MB", () => {
const prev = process.env[envKey]
delete process.env[envKey]
try {
expect(resolveMaxResponseSize()).toBe(DEFAULT_MAX_RESPONSE_SIZE)
expect(resolveMaxResponseSize({})).toBe(DEFAULT_MAX_RESPONSE_SIZE)
} finally {
if (prev === undefined) delete process.env[envKey]
else process.env[envKey] = prev
}
})

test("prefers configured max_response_size", () => {
const prev = process.env[envKey]
process.env[envKey] = "4096"
try {
expect(resolveMaxResponseSize({ webfetch: { max_response_size: 1024 } })).toBe(1024)
} finally {
if (prev === undefined) delete process.env[envKey]
else process.env[envKey] = prev
}
})

test("uses OPENCODE_WEBFETCH_MAX_SIZE when config is unset", () => {
const prev = process.env[envKey]
process.env[envKey] = "20971520"
try {
expect(resolveMaxResponseSize()).toBe(20971520)
expect(resolveMaxResponseSize({ webfetch: {} })).toBe(20971520)
} finally {
if (prev === undefined) delete process.env[envKey]
else process.env[envKey] = prev
}
})

test("ignores invalid environment values", () => {
const prev = process.env[envKey]
try {
for (const invalid of ["", "0", "-1", "1.5", "abc", "1024abc"]) {
process.env[envKey] = invalid
expect(resolveMaxResponseSize()).toBe(DEFAULT_MAX_RESPONSE_SIZE)
}
} finally {
if (prev === undefined) delete process.env[envKey]
else process.env[envKey] = prev
}
})
})
Loading