Web-search connector (Google CSE) + extract_contacts step — enrichment (#99)

workers/api/src/lib/connectors/registry.test.ts

@@ -2,9 +2,18 @@ import { describe, expect, it } from "vitest";
 import { CONNECTORS, connectorTools, getConnector } from "./registry.js";
 
 describe("connector registry", () => {
-	it("declares github, http, meta, and tmux", () => {
+	it("declares github, http, meta, tmux, and web-search", () => {
 		const ids = CONNECTORS.map((c) => c.id).sort();
-		expect(ids).toEqual(["github", "http", "meta", "tmux"]);
+		expect(ids).toEqual(["github", "http", "meta", "tmux", "web-search"]);
+	});
+
+	it("web-search is a token-auth, read-only, user-grant connector with no tokenEnv (vault-backed)", () => {
+		const ws = getConnector("web-search");
+		expect(ws?.auth).toBe("token");
+		expect(ws?.tokenEnv).toBeUndefined(); // no platform env → connectorClient reads the vault key
+		expect(ws?.scopes).toEqual({ read: true, write: false });
+		expect(ws?.grantModel).toBe("user");
+		expect(ws?.tools.map((t) => t.name)).toEqual(["web_search"]);
 	});
 
 	it("http is a token-auth, read+write, user-grant connector with no tokenEnv (vault-backed)", () => {

workers/api/src/lib/connectors/registry.ts

@@ -8,6 +8,7 @@ import { GITHUB_TOOLS } from "./github.js";
 import { HTTP_TOOLS } from "./http.js";
 import { META_TOOLS } from "./meta.js";
 import { TMUX_TOOLS } from "./tmux.js";
+import { WEB_SEARCH_TOOLS } from "./web-search.js";
 
 export interface Connector {
 	/** Stable id, also the `connector` stamped on its tools and the grants/consent key. */
@@ -80,6 +81,17 @@ export const CONNECTORS: Connector[] = [
 		grantModel: "user",
 		tools: HTTP_TOOLS,
 	},
+	{
+		id: "web-search",
+		label: "Web Search (Google Custom Search)",
+		// auth:"token", no tokenEnv → connectorClient.token() reads the user's vault key
+		// (user_api_keys, provider "web-search") — a separate slot from the http connector's
+		// key. web_search injects it into the request URL itself. Read-only (search only).
+		auth: "token",
+		scopes: { read: true, write: false },
+		grantModel: "user",
+		tools: WEB_SEARCH_TOOLS,
+	},
 ];
 
 const BY_ID: ReadonlyMap<string, Connector> = new Map(CONNECTORS.map((c) => [c.id, c] as const));

workers/api/src/lib/connectors/web-search.test.ts

@@ -0,0 +1,180 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { getRegistryTool, runRegistryTool } from "../tool-registry.js";
+import type { RegistryToolCtx } from "../tool-registry.js";
+import type { ConnectorClient } from "./client.js";
+
+// web_search resolved from the REGISTRY (proves it's registered → callable via runtime,
+// MCP proxy, and POST …/tools/web_search with no bespoke route).
+const webSearch = getRegistryTool("web_search")!;
+
+/** Mock globalThis.fetch (what safeFetch calls). Records the URL + init. */
+function mockFetch(status: number, body: unknown) {
+	const calls: Array<{ url: string; init: RequestInit }> = [];
+	vi.spyOn(globalThis, "fetch").mockImplementation(async (url: any, init: any) => {
+		calls.push({ url: String(url), init: init || {} });
+		return new Response(typeof body === "string" ? body : JSON.stringify(body), {
+			status,
+			headers: { "Content-Type": "application/json" },
+		});
+	});
+	return { calls };
+}
+
+/** A ctx whose connectorClient("web-search").token() returns the vault key + a cx env. */
+function ctxWithKey(key: string, cx = "CSE_ID"): RegistryToolCtx {
+	const client = { token: async () => key } as unknown as ConnectorClient;
+	return { env: { WEB_SEARCH_CX: cx } as any, connectorClient: () => client } as RegistryToolCtx;
+}
+
+function parse(content: string): any {
+	try {
+		return JSON.parse(content);
+	} catch {
+		return undefined;
+	}
+}
+
+afterEach(() => vi.restoreAllMocks());
+
+// A sample Google Custom Search JSON response for "Blue Bottle Cafe Newtown".
+const CSE_RESPONSE = {
+	items: [
+		{ title: "Blue Bottle Cafe (@bluebottle_syd) • Instagram photos", link: "https://www.instagram.com/bluebottle_syd/", snippet: "Newtown, Sydney NSW." },
+		{ title: "Blue Bottle Cafe - Home | Facebook", link: "https://www.facebook.com/BlueBottleNewtown", snippet: "Cafe · Newtown. Email hello@bluebottle.example" },
+		{ title: "Blue Bottle Cafe", link: "https://bluebottle.example", snippet: "Specialty coffee in Newtown." },
+	],
+};
+
+describe("web_search — registration & schema", () => {
+	it("is registered as a web-search-connector, read-scoped tool", () => {
+		expect(webSearch.connector).toBe("web-search");
+		expect(webSearch.tier).toBe("connector");
+		expect(webSearch.scope).toBe("read");
+	});
+	it("exposes query/num/cx in its schema and requires query", () => {
+		const p = webSearch.jsonSchema.properties;
+		for (const k of ["query", "num", "cx"]) expect(p[k]).toBeDefined();
+		expect(webSearch.jsonSchema.required).toContain("query");
+	});
+	it("errors (not throws) when query is empty", async () => {
+		const r = await webSearch.handler(ctxWithKey("K"), { query: "" });
+		expect(r.success).toBe(false);
+		expect(r.content).toMatch(/query/i);
+	});
+});
+
+describe("web_search — Google Custom Search wiring", () => {
+	it("returns [{title, link, snippet}] for a business query", async () => {
+		mockFetch(200, CSE_RESPONSE);
+		const r = await webSearch.handler(ctxWithKey("SECRET_KEY"), { query: "Blue Bottle Cafe Newtown", num: 3 });
+		expect(r.success).toBe(true);
+		const out = parse(r.content);
+		expect(out.count).toBe(3);
+		expect(out.results[0]).toEqual({
+			title: "Blue Bottle Cafe (@bluebottle_syd) • Instagram photos",
+			link: "https://www.instagram.com/bluebottle_syd/",
+			snippet: "Newtown, Sydney NSW.",
+		});
+	});
+	it("hits the Custom Search endpoint with q, cx, and num", async () => {
+		const { calls } = mockFetch(200, CSE_RESPONSE);
+		await webSearch.handler(ctxWithKey("SECRET_KEY", "MY_CSE"), { query: "cafe newtown", num: 2 });
+		const u = new URL(calls[0].url);
+		expect(u.origin + u.pathname).toBe("https://www.googleapis.com/customsearch/v1");
+		expect(u.searchParams.get("q")).toBe("cafe newtown");
+		expect(u.searchParams.get("cx")).toBe("MY_CSE");
+		expect(u.searchParams.get("num")).toBe("2");
+	});
+	it("a per-call cx overrides the WEB_SEARCH_CX env default", async () => {
+		const { calls } = mockFetch(200, CSE_RESPONSE);
+		await webSearch.handler(ctxWithKey("K", "ENV_CX"), { query: "x", cx: "CALL_CX" });
+		expect(new URL(calls[0].url).searchParams.get("cx")).toBe("CALL_CX");
+	});
+	it("caps num at 10 and defaults to 5", async () => {
+		const { calls } = mockFetch(200, { items: [] });
+		await webSearch.handler(ctxWithKey("K"), { query: "x", num: 99 });
+		expect(new URL(calls[0].url).searchParams.get("num")).toBe("10");
+		await webSearch.handler(ctxWithKey("K"), { query: "y" });
+		expect(new URL(calls[1].url).searchParams.get("num")).toBe("5");
+	});
+	it("no CSE id (no cx, no env) → clean failure, no request", async () => {
+		const { calls } = mockFetch(200, {});
+		const noCx = { env: {} as any, connectorClient: () => ({ token: async () => "K" }) as any } as RegistryToolCtx;
+		const r = await webSearch.handler(noCx, { query: "x" });
+		expect(r.success).toBe(false);
+		expect(r.content).toMatch(/cx|WEB_SEARCH_CX/i);
+		expect(calls).toHaveLength(0);
+	});
+});
+
+describe("web_search — api-key from vault, never leaked", () => {
+	it("reads the key via connectorClient and injects it into the request URL", async () => {
+		const { calls } = mockFetch(200, CSE_RESPONSE);
+		await webSearch.handler(ctxWithKey("VAULT_KEY"), { query: "x" });
+		expect(new URL(calls[0].url).searchParams.get("key")).toBe("VAULT_KEY");
+	});
+	it("never echoes the key into the returned result", async () => {
+		mockFetch(200, CSE_RESPONSE);
+		const r = await webSearch.handler(ctxWithKey("VAULT_KEY"), { query: "x" });
+		expect(r.content).not.toContain("VAULT_KEY");
+	});
+	it("exposes no key input in the schema (the value comes from the vault, not the caller)", () => {
+		// The caller can never SUPPLY the key: there is no `key`/`apiKey`/`token` input property.
+		// (The description documenting "read from the vault" is intentional; the VALUE is never here.)
+		for (const forbidden of ["key", "apiKey", "api_key", "token", "secret"]) {
+			expect(webSearch.jsonSchema.properties[forbidden]).toBeUndefined();
+		}
+		expect(Object.keys(webSearch.jsonSchema.properties).sort()).toEqual(["cx", "num", "query"]);
+	});
+	it("fails cleanly (no request) when no key is connected", async () => {
+		const { calls } = mockFetch(200, {});
+		const noKey = { env: { WEB_SEARCH_CX: "CSE" } as any, connectorClient: () => ({ token: async () => "" }) as any } as RegistryToolCtx;
+		const r = await webSearch.handler(noKey, { query: "x" });
+		expect(r.success).toBe(false);
+		expect(calls).toHaveLength(0);
+	});
+	it("an upstream error does not leak the request URL (which carries the key)", async () => {
+		mockFetch(403, { error: { message: "quota" } });
+		const r = await webSearch.handler(ctxWithKey("VAULT_KEY"), { query: "x" });
+		expect(r.success).toBe(false);
+		expect(r.content).not.toContain("VAULT_KEY");
+	});
+});
+
+describe("web_search — SSRF safety (uses safeFetch)", () => {
+	it("only ever targets the Google endpoint (public https) — a mocked SSRF host is impossible via input", async () => {
+		// web_search builds the URL itself from the fixed endpoint, so a caller can't redirect it
+		// to a private target. Confirm the outbound URL host is always googleapis.com.
+		const { calls } = mockFetch(200, { items: [] });
+		await webSearch.handler(ctxWithKey("K"), { query: "http://169.254.169.254/latest/meta-data" });
+		expect(new URL(calls[0].url).hostname).toBe("www.googleapis.com");
+	});
+	it("goes through safeFetch — a blocked target surfaces as a failure, not a throw", async () => {
+		// Simulate safeFetch rejecting (e.g. a poisoned redirect) by making the underlying fetch throw.
+		vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("boom"));
+		const r = await webSearch.handler(ctxWithKey("K"), { query: "x" });
+		expect(r.success).toBe(false);
+		expect(r.content).toMatch(/search failed/i);
+	});
+});
+
+// ── the #99 acceptance proof: business name+suburb → instagram/facebook/email as config ──
+describe("web_search + extract_contacts — enrichment (issue #99 acceptance)", () => {
+	it("given a business name+suburb, yields instagram/facebook/email columns (pure config)", async () => {
+		mockFetch(200, CSE_RESPONSE);
+		const ctx = ctxWithKey("VAULT_KEY");
+
+		// 1) search the web for the business (dispatched through runRegistryTool → audited/granted).
+		const search = await runRegistryTool("web_search", ctx, { query: "Blue Bottle Cafe Newtown Sydney" });
+		expect(search.success).toBe(true);
+
+		// 2) extract socials + email from the results — no bespoke code, just the step catalog.
+		const enriched = await runRegistryTool("extract_contacts", ctx, { items: parse(search.content) });
+		expect(enriched.success).toBe(true);
+
+		const cols = parse(enriched.content);
+		expect(cols.instagram).toBe("https://instagram.com/bluebottle_syd");
+		expect(cols.facebook).toBe("https://facebook.com/BlueBottleNewtown");
+		expect(cols.email).toBe("hello@bluebottle.example");
+	});
+});

workers/api/src/lib/connectors/web-search.ts

@@ -0,0 +1,105 @@
+// Web-search connector (issue #99, child of #94/#84). Enriching a lead with its socials +
+// email needs a web search — a capability the generic `http` connector can't cleanly cover
+// because the search KEY must live in its OWN vault slot (provider "web-search"), separate
+// from the http connector's key. So this is a first-class connector: declared once in the
+// registry, it provides ONE `web_search` tool.
+//
+// Auth (declared on the connector as auth:"token", grantModel:"user", no tokenEnv):
+//   the Google Custom Search API KEY is read from the vault (user_api_keys, provider
+//   "web-search") via ctx.connectorClient("web-search").token(). The key value is used only
+//   on the wire — it NEVER appears in the tool inputs, schema, or returned result.
+//
+// The CSE id (`cx`) is NOT a secret (it identifies the search engine, not the account), so it
+// is a plain tool input (or the WEB_SEARCH_CX env default), not a vault secret.
+//
+// Every request goes through safeFetch (lib/ssrf.ts) — https-only, SSRF-guarded, redirects
+// re-validated — the same guard the #95 http connector uses. We do NOT re-implement fetch/SSRF.
+import type { ToolDef, RegistryToolCtx } from "../tool-registry.js";
+import { safeFetch, SsrfError } from "../ssrf.js";
+
+// Google Custom Search JSON API endpoint. Wrapped, not hardcoded into agent code:
+//   GET https://www.googleapis.com/customsearch/v1?key=<vault>&cx=<cse-id>&q=<query>&num=<n>
+const CSE_ENDPOINT = "https://www.googleapis.com/customsearch/v1";
+
+// Per-call result cap. Google CSE returns at most 10 items per request; we also cap the
+// caller-supplied `num` so a single web_search call can never fan out beyond one page.
+const MAX_RESULTS = 10;
+const DEFAULT_RESULTS = 5;
+
+interface CseItem {
+	title?: string;
+	link?: string;
+	snippet?: string;
+}
+
+export const WEB_SEARCH_TOOLS: ToolDef[] = [
+	{
+		name: "web_search",
+		tier: "connector",
+		connector: "web-search",
+		scope: "read",
+		description:
+			"Search the web (Google Custom Search) for a query and return the top results as [{title, link, snippet}]. Use it to look a business up by name+suburb before an extract step pulls its Instagram/Facebook/email. The API key is read from the vault (never passed in); supply the non-secret CSE id via `cx` (or the WEB_SEARCH_CX env default). `num` caps results (default 5, max 10). Results are best-effort. HTTPS-only, SSRF-guarded.",
+		jsonSchema: {
+			type: "object",
+			properties: {
+				query: { type: "string", description: "The search query, e.g. 'Blue Bottle Cafe Newtown Sydney instagram'." },
+				num: { type: "number", description: "Number of results to return (default 5, max 10)." },
+				cx: { type: "string", description: "Google Custom Search engine id (cx). Not a secret — defaults to the WEB_SEARCH_CX env if omitted." },
+			},
+			required: ["query"],
+		},
+		handler: async (ctx: RegistryToolCtx, input) => {
+			const query = String(input.query ?? "").trim();
+			if (!query) return { content: "query is required.", success: false };
+
+			// cx: non-secret search-engine id. Tool input takes precedence, else the env default.
+			const cx = (typeof input.cx === "string" && input.cx.trim()) || ctx.env?.WEB_SEARCH_CX?.trim() || "";
+			if (!cx) return { content: "No Custom Search engine id — pass `cx` or set WEB_SEARCH_CX.", success: false };
+
+			const num = Math.max(1, Math.min(Number(input.num) || DEFAULT_RESULTS, MAX_RESULTS));
+
+			// The API key comes from the vault via the connectorClient (provider "web-search").
+			// It is attached to the outgoing URL only — never returned or logged.
+			const key = await ctx.connectorClient?.("web-search").token().catch(() => null);
+			if (!key) return { content: "No API key connected for the web-search connector — add one in the instance's Connections settings.", success: false };
+
+			const u = new URL(CSE_ENDPOINT);
+			u.searchParams.set("key", key);
+			u.searchParams.set("cx", cx);
+			u.searchParams.set("q", query);
+			u.searchParams.set("num", String(num));
+
+			let res: Response;
+			try {
+				res = await safeFetch(u.toString(), { method: "GET" });
+			} catch (e) {
+				if (e instanceof SsrfError) return { content: `Blocked: ${e.message}`, success: false };
+				return { content: `Search failed: ${e instanceof Error ? e.message : String(e)}`, success: false };
+			}
+
+			const text = await res.text();
+			let body: unknown = text;
+			try {
+				body = JSON.parse(text);
+			} catch {
+				/* keep as text */
+			}
+			if (!res.ok) {
+				// Surface the status but NOT the request URL (which carries the key).
+				const msg = body && typeof body === "object" ? JSON.stringify((body as { error?: unknown }).error ?? body) : String(body);
+				return { content: `Search error ${res.status}: ${msg}`.slice(0, 500), success: false };
+			}
+
+			const rawItems = (body && typeof body === "object" ? (body as { items?: unknown }).items : undefined);
+			const items: CseItem[] = Array.isArray(rawItems) ? rawItems : [];
+			const results = items.slice(0, num).map((it) => ({
+				title: typeof it.title === "string" ? it.title : "",
+				link: typeof it.link === "string" ? it.link : "",
+				snippet: typeof it.snippet === "string" ? it.snippet : "",
+			}));
+
+			return { content: JSON.stringify({ query, count: results.length, results }, null, 2), success: true };
+		},
+	},
+];