diff --git a/src/signals/focus-manifest-loader.ts b/src/signals/focus-manifest-loader.ts index ade1aa4361..776d32a927 100644 --- a/src/signals/focus-manifest-loader.ts +++ b/src/signals/focus-manifest-loader.ts @@ -1,10 +1,11 @@ import { listSignalSnapshots, persistSignalSnapshot } from "../db/repositories"; import type { JsonValue } from "../types"; import { nowIso } from "../utils/json"; -import { parseFocusManifest, parseFocusManifestContent, type FocusManifest, type FocusManifestSource } from "./focus-manifest"; +import { MAX_FOCUS_MANIFEST_BYTES, parseFocusManifest, parseFocusManifestContent, type FocusManifest, type FocusManifestSource } from "./focus-manifest"; export const REPO_FOCUS_MANIFEST_SIGNAL = "repo-focus-manifest"; export const REPO_FOCUS_MANIFEST_MAX_AGE_MS = 6 * 60 * 60 * 1000; +export const REPO_FOCUS_MANIFEST_MAX_CONCURRENT_LOADS = 4; /** * Async source for the raw manifest text of a single repo. Returns null when no manifest is @@ -27,7 +28,10 @@ export async function fetchRepoFocusManifestFile(repoFullName: string): Promise< const url = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/HEAD/${path}`; try { const response = await fetch(url, { headers: { Accept: "application/json", "User-Agent": "gittensory" } }); - if (response.ok) return await response.text(); + if (response.ok) { + const text = await readBoundedResponseText(response); + if (text !== null) return text; + } } catch { // try the next candidate path } @@ -71,12 +75,56 @@ export async function loadRepoFocusManifests( repoFullNames: string[], options: { fetcher?: RepoFocusManifestFetcher; maxAgeMs?: number } = {}, ): Promise> { - const entries = await Promise.all( - repoFullNames.map(async (name) => [name.toLowerCase(), await loadRepoFocusManifest(env, name, options)] as const), + const entries = await mapWithConcurrencyLimit(repoFullNames, REPO_FOCUS_MANIFEST_MAX_CONCURRENT_LOADS, async (name) => + [name.toLowerCase(), await loadRepoFocusManifest(env, name, options)] as const, ); return new Map(entries); } +async function readBoundedResponseText(response: Response): Promise { + const contentLength = response.headers.get("content-length"); + if (contentLength !== null) { + const parsedLength = Number.parseInt(contentLength, 10); + if (Number.isFinite(parsedLength) && parsedLength > MAX_FOCUS_MANIFEST_BYTES) return null; + } + if (!response.body) return ""; + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let totalBytes = 0; + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > MAX_FOCUS_MANIFEST_BYTES) { + await reader.cancel(); + return null; + } + text += decoder.decode(value, { stream: true }); + } + text += decoder.decode(); + return text; + } finally { + reader.releaseLock(); + } +} + +async function mapWithConcurrencyLimit(items: T[], limit: number, mapper: (item: T) => Promise): Promise { + const results: U[] = new Array(items.length); + let nextIndex = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await mapper(items[index]!); + } + }); + await Promise.all(workers); + return results; +} + /** * Persist a maintainer-supplied manifest (e.g. from a maintainer API/console) so subsequent * decision-pack and branch-analysis paths pick it up without refetching the repo file. diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 3b4af94d03..6af65c07a8 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -57,6 +57,7 @@ export type FocusManifestGuidance = { const MAX_LIST_ITEMS = 200; const MAX_ITEM_LENGTH = 300; +export const MAX_FOCUS_MANIFEST_BYTES = 64 * 1024; const EMPTY_MANIFEST: FocusManifest = { present: false, @@ -173,6 +174,9 @@ export function parseFocusManifest(raw: unknown, source?: FocusManifestSource): */ export function parseFocusManifestContent(content: string | null | undefined, source: FocusManifestSource = "repo_file"): FocusManifest { if (content === undefined || content === null || content.trim() === "") return emptyManifest(source); + if (content.length > MAX_FOCUS_MANIFEST_BYTES || new TextEncoder().encode(content).byteLength > MAX_FOCUS_MANIFEST_BYTES) { + return emptyManifest(source, [`Manifest content exceeded ${MAX_FOCUS_MANIFEST_BYTES} bytes; ignoring it and falling back to deterministic signals.`]); + } let parsed: unknown; try { parsed = JSON.parse(content); diff --git a/test/unit/focus-manifest-loader.test.ts b/test/unit/focus-manifest-loader.test.ts index 470dce480b..c093aa28ca 100644 --- a/test/unit/focus-manifest-loader.test.ts +++ b/test/unit/focus-manifest-loader.test.ts @@ -6,7 +6,9 @@ import { loadRepoFocusManifests, upsertRepoFocusManifest, REPO_FOCUS_MANIFEST_MAX_AGE_MS, + REPO_FOCUS_MANIFEST_MAX_CONCURRENT_LOADS, } from "../../src/signals/focus-manifest-loader"; +import { MAX_FOCUS_MANIFEST_BYTES, parseFocusManifestContent } from "../../src/signals/focus-manifest"; describe("focus-manifest loader", () => { afterEach(() => vi.restoreAllMocks()); @@ -84,18 +86,28 @@ describe("focus-manifest loader", () => { expect(reloaded.source).toBe("api_record"); }); - it("bulk-loads manifests for many repos in parallel", async () => { + it("bulk-loads manifests for many repos with a concurrency cap", async () => { const env = createTestEnv(); - const fetcher = async (repoFullName: string) => - repoFullName === "owner/a" + let active = 0; + let maxActive = 0; + const fetcher = async (repoFullName: string) => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active -= 1; + return repoFullName === "owner/a" ? JSON.stringify({ wantedPaths: ["src/"] }) : repoFullName === "owner/b" ? JSON.stringify({ blockedPaths: ["dist/"] }) : null; - const map = await loadRepoFocusManifests(env, ["owner/a", "owner/b", "owner/c"], { fetcher }); + }; + const repos = ["owner/a", "owner/b", "owner/c", "owner/d", "owner/e", "owner/f"]; + const map = await loadRepoFocusManifests(env, repos, { fetcher }); expect(map.get("owner/a")?.wantedPaths).toEqual(["src/"]); expect(map.get("owner/b")?.blockedPaths).toEqual(["dist/"]); expect(map.get("owner/c")?.present).toBe(false); + expect(maxActive).toBeGreaterThan(1); + expect(maxActive).toBeLessThanOrEqual(REPO_FOCUS_MANIFEST_MAX_CONCURRENT_LOADS); }); it("rejects an invalid repoFullName from the public fetcher without throwing", async () => { @@ -116,6 +128,44 @@ describe("focus-manifest loader", () => { expect(fetchSpy).toHaveBeenCalledTimes(2); }); + it("does not read public manifest responses when Content-Length is too large", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async (url) => { + const stringUrl = String(url); + if (stringUrl.endsWith("/.gittensory.json")) { + return new Response('{"wantedPaths":["too-large/"]}', { + status: 200, + headers: { "content-length": String(MAX_FOCUS_MANIFEST_BYTES + 1) }, + }); + } + return new Response('{"wantedPaths":["src/"]}', { status: 200 }); + }); + const text = await fetchRepoFocusManifestFile("owner/repo"); + expect(text).toBe('{"wantedPaths":["src/"]}'); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); + + it("aborts public manifest streams that grow beyond the byte cap", async () => { + const oversizedChunk = new Uint8Array(MAX_FOCUS_MANIFEST_BYTES + 1); + vi.spyOn(globalThis, "fetch").mockImplementation(async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(oversizedChunk); + controller.close(); + }, + }), + { status: 200 }, + ), + ); + expect(await fetchRepoFocusManifestFile("owner/repo")).toBeNull(); + }); + + it("rejects oversized raw manifest content before JSON parsing", () => { + const manifest = parseFocusManifestContent(`{ "wantedPaths": ["${"a".repeat(MAX_FOCUS_MANIFEST_BYTES)}"] }`); + expect(manifest.present).toBe(false); + expect(manifest.warnings.join(" ")).toMatch(/exceeded/); + }); + it("returns null when every candidate path responds non-ok", async () => { vi.spyOn(globalThis, "fetch").mockImplementation(async () => new Response("nope", { status: 404 })); expect(await fetchRepoFocusManifestFile("owner/repo")).toBeNull();