From fa899a4ddf4438d70da55e477810fa270667a106 Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:44:54 +0000 Subject: [PATCH 1/4] Log per-action tool calls in CuaAgent templates The anthropic/openai/gemini computer-use templates never subscribed to CuaAgent's tool_execution_start/end events, so they only printed the final answer. Wire up agent.subscribe() with a small logger so each click/type/screenshot action streams to stdout, matching the yutori and tzafon templates. --- .../anthropic-computer-use/index.ts | 2 ++ .../anthropic-computer-use/logging.ts | 19 +++++++++++++++++++ .../typescript/gemini-computer-use/index.ts | 2 ++ .../typescript/gemini-computer-use/logging.ts | 19 +++++++++++++++++++ .../typescript/openai-computer-use/index.ts | 2 ++ .../openai-computer-use/lib/logging.ts | 19 +++++++++++++++++++ 6 files changed, 63 insertions(+) create mode 100644 pkg/templates/typescript/anthropic-computer-use/logging.ts create mode 100644 pkg/templates/typescript/gemini-computer-use/logging.ts create mode 100644 pkg/templates/typescript/openai-computer-use/lib/logging.ts diff --git a/pkg/templates/typescript/anthropic-computer-use/index.ts b/pkg/templates/typescript/anthropic-computer-use/index.ts index f3c7e4d3..d501fbc8 100644 --- a/pkg/templates/typescript/anthropic-computer-use/index.ts +++ b/pkg/templates/typescript/anthropic-computer-use/index.ts @@ -2,6 +2,7 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; import { CuaAgent } from '@onkernel/cua-agent'; import type { AssistantMessage } from '@onkernel/cua-ai'; import { KernelBrowserSession } from './session'; +import { logAgentEvent } from './logging'; const kernel = new Kernel(); @@ -82,6 +83,7 @@ app.action( systemPrompt: SYSTEM_PROMPT, }, }); + agent.subscribe(logAgentEvent); await agent.prompt(payload.query); diff --git a/pkg/templates/typescript/anthropic-computer-use/logging.ts b/pkg/templates/typescript/anthropic-computer-use/logging.ts new file mode 100644 index 00000000..9979a3da --- /dev/null +++ b/pkg/templates/typescript/anthropic-computer-use/logging.ts @@ -0,0 +1,19 @@ +import type { AgentEvent } from '@onkernel/cua-agent'; + +// Logs each computer-use tool call as CuaAgent executes it. Pass to +// `agent.subscribe(logAgentEvent)` to see every action (click, type, +// screenshot, ...) as it happens instead of only the final answer. +export function logAgentEvent(event: AgentEvent): void { + if (event.type === 'tool_execution_start') { + console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); + return; + } + if (event.type === 'tool_execution_end') { + console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + } +} + +function formatJson(value: unknown): string { + const text = JSON.stringify(value) ?? 'undefined'; + return text.length > 500 ? `${text.slice(0, 497)}...` : text; +} diff --git a/pkg/templates/typescript/gemini-computer-use/index.ts b/pkg/templates/typescript/gemini-computer-use/index.ts index 9433d66c..a8a0f8c4 100644 --- a/pkg/templates/typescript/gemini-computer-use/index.ts +++ b/pkg/templates/typescript/gemini-computer-use/index.ts @@ -2,6 +2,7 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; import { CuaAgent } from '@onkernel/cua-agent'; import type { AssistantMessage } from '@onkernel/cua-ai'; import { KernelBrowserSession } from './session'; +import { logAgentEvent } from './logging'; const kernel = new Kernel(); @@ -67,6 +68,7 @@ The current date is ${currentDate}.`; systemPrompt, }, }); + agent.subscribe(logAgentEvent); await agent.prompt(payload.query); diff --git a/pkg/templates/typescript/gemini-computer-use/logging.ts b/pkg/templates/typescript/gemini-computer-use/logging.ts new file mode 100644 index 00000000..9979a3da --- /dev/null +++ b/pkg/templates/typescript/gemini-computer-use/logging.ts @@ -0,0 +1,19 @@ +import type { AgentEvent } from '@onkernel/cua-agent'; + +// Logs each computer-use tool call as CuaAgent executes it. Pass to +// `agent.subscribe(logAgentEvent)` to see every action (click, type, +// screenshot, ...) as it happens instead of only the final answer. +export function logAgentEvent(event: AgentEvent): void { + if (event.type === 'tool_execution_start') { + console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); + return; + } + if (event.type === 'tool_execution_end') { + console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + } +} + +function formatJson(value: unknown): string { + const text = JSON.stringify(value) ?? 'undefined'; + return text.length > 500 ? `${text.slice(0, 497)}...` : text; +} diff --git a/pkg/templates/typescript/openai-computer-use/index.ts b/pkg/templates/typescript/openai-computer-use/index.ts index d18e9537..04bdebc8 100644 --- a/pkg/templates/typescript/openai-computer-use/index.ts +++ b/pkg/templates/typescript/openai-computer-use/index.ts @@ -2,6 +2,7 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; import { CuaAgent } from '@onkernel/cua-agent'; import type { AssistantMessage } from '@onkernel/cua-ai'; import { maybeStartReplay, maybeStopReplay } from './lib/replay'; +import { logAgentEvent } from './lib/logging'; const kernel = new Kernel(); const app = kernel.app('ts-openai-cua'); @@ -52,6 +53,7 @@ app.action( systemPrompt: `You are operating a Chromium browser on a Kernel cloud VM. Use the navigation tool to open URLs directly, and review the screenshot after each action before continuing. The current date and time is ${new Date().toISOString()}.`, }, }); + agent.subscribe(logAgentEvent); await agent.prompt(payload.task); diff --git a/pkg/templates/typescript/openai-computer-use/lib/logging.ts b/pkg/templates/typescript/openai-computer-use/lib/logging.ts new file mode 100644 index 00000000..9979a3da --- /dev/null +++ b/pkg/templates/typescript/openai-computer-use/lib/logging.ts @@ -0,0 +1,19 @@ +import type { AgentEvent } from '@onkernel/cua-agent'; + +// Logs each computer-use tool call as CuaAgent executes it. Pass to +// `agent.subscribe(logAgentEvent)` to see every action (click, type, +// screenshot, ...) as it happens instead of only the final answer. +export function logAgentEvent(event: AgentEvent): void { + if (event.type === 'tool_execution_start') { + console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); + return; + } + if (event.type === 'tool_execution_end') { + console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + } +} + +function formatJson(value: unknown): string { + const text = JSON.stringify(value) ?? 'undefined'; + return text.length > 500 ? `${text.slice(0, 497)}...` : text; +} From b9b6e1074f874b74c00ccbdabde3e39e144378e1 Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:57:05 +0000 Subject: [PATCH 2/4] Log tool result details instead of full result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AgentToolResult.content carries the image bytes sent back to the model, so logging the whole result dumped truncated base64 on every screenshot turn. Log result.details — the structured, human-readable summary — to match the browser-loop reference logger. --- pkg/templates/typescript/anthropic-computer-use/logging.ts | 5 ++++- pkg/templates/typescript/gemini-computer-use/logging.ts | 5 ++++- pkg/templates/typescript/openai-computer-use/lib/logging.ts | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/templates/typescript/anthropic-computer-use/logging.ts b/pkg/templates/typescript/anthropic-computer-use/logging.ts index 9979a3da..2090e0d2 100644 --- a/pkg/templates/typescript/anthropic-computer-use/logging.ts +++ b/pkg/templates/typescript/anthropic-computer-use/logging.ts @@ -9,7 +9,10 @@ export function logAgentEvent(event: AgentEvent): void { return; } if (event.type === 'tool_execution_end') { - console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + // Log the structured `details` rather than the full result: `result.content` + // is what's sent back to the model and includes screenshot image bytes. + const details = (event.result as { details?: unknown } | undefined)?.details; + console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); } } diff --git a/pkg/templates/typescript/gemini-computer-use/logging.ts b/pkg/templates/typescript/gemini-computer-use/logging.ts index 9979a3da..2090e0d2 100644 --- a/pkg/templates/typescript/gemini-computer-use/logging.ts +++ b/pkg/templates/typescript/gemini-computer-use/logging.ts @@ -9,7 +9,10 @@ export function logAgentEvent(event: AgentEvent): void { return; } if (event.type === 'tool_execution_end') { - console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + // Log the structured `details` rather than the full result: `result.content` + // is what's sent back to the model and includes screenshot image bytes. + const details = (event.result as { details?: unknown } | undefined)?.details; + console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); } } diff --git a/pkg/templates/typescript/openai-computer-use/lib/logging.ts b/pkg/templates/typescript/openai-computer-use/lib/logging.ts index 9979a3da..2090e0d2 100644 --- a/pkg/templates/typescript/openai-computer-use/lib/logging.ts +++ b/pkg/templates/typescript/openai-computer-use/lib/logging.ts @@ -9,7 +9,10 @@ export function logAgentEvent(event: AgentEvent): void { return; } if (event.type === 'tool_execution_end') { - console.log(`[tool:end] ${event.toolName} error=${event.isError} result=${formatJson(event.result)}`); + // Log the structured `details` rather than the full result: `result.content` + // is what's sent back to the model and includes screenshot image bytes. + const details = (event.result as { details?: unknown } | undefined)?.details; + console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); } } From a9bb98ed46131622afc07ce23b750a0ce2bf1d1d Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:32:33 +0000 Subject: [PATCH 3/4] Log agent reasoning and readable actions, not raw tool events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first pass logged raw tool_execution_start/end pairs with screenshot byte counts — hard to follow. Subscribe to message_end too and print the model's narration (`agent>`) between steps, then each action as a concise line (`→ click (85, 267)`, `→ drag (780,385) → (1100,385)`), unwrapping Anthropic's computer_batch and OpenAI's computer_use_extra so every provider reads the same. Failed actions are marked so retries are visible. --- .../anthropic-computer-use/logging.ts | 105 +++++++++++++++--- .../typescript/gemini-computer-use/logging.ts | 105 +++++++++++++++--- .../openai-computer-use/lib/logging.ts | 105 +++++++++++++++--- 3 files changed, 273 insertions(+), 42 deletions(-) diff --git a/pkg/templates/typescript/anthropic-computer-use/logging.ts b/pkg/templates/typescript/anthropic-computer-use/logging.ts index 2090e0d2..d1f5ad93 100644 --- a/pkg/templates/typescript/anthropic-computer-use/logging.ts +++ b/pkg/templates/typescript/anthropic-computer-use/logging.ts @@ -1,22 +1,99 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Logs each computer-use tool call as CuaAgent executes it. Pass to -// `agent.subscribe(logAgentEvent)` to see every action (click, type, -// screenshot, ...) as it happens instead of only the final answer. +// Human-readable trace of what the agent is doing, meant for +// `agent.subscribe(logAgentEvent)`: the model's narration between steps +// (`agent>`), then each concrete browser action it takes (`→`). Failed +// actions are marked so retries are visible. export function logAgentEvent(event: AgentEvent): void { - if (event.type === 'tool_execution_start') { - console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); - return; + switch (event.type) { + case 'message_end': { + const text = assistantText(event.message); + if (text) console.log(`\nagent> ${text}`); + return; + } + case 'tool_execution_start': + console.log(` → ${formatAction(event.toolName, event.args)}`); + return; + case 'tool_execution_end': + if (event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } + return; } - if (event.type === 'tool_execution_end') { - // Log the structured `details` rather than the full result: `result.content` - // is what's sent back to the model and includes screenshot image bytes. - const details = (event.result as { details?: unknown } | undefined)?.details; - console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); +} + +// Concatenate the visible text blocks of an assistant message. Non-assistant +// messages and tool-call-only turns have no narration and return ''. +function assistantText(message: unknown): string { + const m = message as { role?: string; content?: unknown }; + if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; + return m.content + .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) + .map((b) => b.text.trim()) + .filter(Boolean) + .join(' '); +} + +function isTextBlock(b: unknown): boolean { + return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; +} + +// Render one tool call as a short action line. Anthropic batches actions under +// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow +// through here — batch sub-actions carry their kind in `type`. +function formatAction(toolName: string, rawArgs: unknown): string { + const a = (rawArgs ?? {}) as Record; + switch (toolName) { + case 'computer_batch': + return Array.isArray(a.actions) + ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') + : 'batch'; + case 'computer_use_extra': + // OpenAI's navigation helper wraps the real action under `action`. + return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); + case 'screenshot': + return 'screenshot'; + case 'goto': + return `goto ${a.url ?? ''}`.trim(); + case 'click': + case 'left_click': + case 'double_click': + case 'right_click': + return `${toolName} ${point(a)}`; + case 'drag': + return `drag ${dragPath(a.path)}`; + case 'keypress': + return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; + case 'type': + return `type ${quote(a.text)}`; + case 'scroll': + return `scroll ${point(a)}`.trim(); + case 'wait': + return `wait ${a.ms ?? ''}ms`; + default: + return `${toolName ?? 'action'} ${compact(a)}`.trim(); } } -function formatJson(value: unknown): string { - const text = JSON.stringify(value) ?? 'undefined'; - return text.length > 500 ? `${text.slice(0, 497)}...` : text; +function point(a: Record): string { + if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; + if (typeof a.x === 'number') return `(${a.x})`; + return ''; +} + +function dragPath(path: unknown): string { + if (!Array.isArray(path) || path.length === 0) return ''; + const first = path[0]; + const last = path[path.length - 1]; + return `${point(first)} → ${point(last)}`; +} + +function quote(text: unknown): string { + const s = typeof text === 'string' ? text : String(text ?? ''); + return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +} + +function compact(value: unknown): string { + const text = JSON.stringify(value) ?? ''; + return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; } diff --git a/pkg/templates/typescript/gemini-computer-use/logging.ts b/pkg/templates/typescript/gemini-computer-use/logging.ts index 2090e0d2..d1f5ad93 100644 --- a/pkg/templates/typescript/gemini-computer-use/logging.ts +++ b/pkg/templates/typescript/gemini-computer-use/logging.ts @@ -1,22 +1,99 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Logs each computer-use tool call as CuaAgent executes it. Pass to -// `agent.subscribe(logAgentEvent)` to see every action (click, type, -// screenshot, ...) as it happens instead of only the final answer. +// Human-readable trace of what the agent is doing, meant for +// `agent.subscribe(logAgentEvent)`: the model's narration between steps +// (`agent>`), then each concrete browser action it takes (`→`). Failed +// actions are marked so retries are visible. export function logAgentEvent(event: AgentEvent): void { - if (event.type === 'tool_execution_start') { - console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); - return; + switch (event.type) { + case 'message_end': { + const text = assistantText(event.message); + if (text) console.log(`\nagent> ${text}`); + return; + } + case 'tool_execution_start': + console.log(` → ${formatAction(event.toolName, event.args)}`); + return; + case 'tool_execution_end': + if (event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } + return; } - if (event.type === 'tool_execution_end') { - // Log the structured `details` rather than the full result: `result.content` - // is what's sent back to the model and includes screenshot image bytes. - const details = (event.result as { details?: unknown } | undefined)?.details; - console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); +} + +// Concatenate the visible text blocks of an assistant message. Non-assistant +// messages and tool-call-only turns have no narration and return ''. +function assistantText(message: unknown): string { + const m = message as { role?: string; content?: unknown }; + if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; + return m.content + .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) + .map((b) => b.text.trim()) + .filter(Boolean) + .join(' '); +} + +function isTextBlock(b: unknown): boolean { + return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; +} + +// Render one tool call as a short action line. Anthropic batches actions under +// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow +// through here — batch sub-actions carry their kind in `type`. +function formatAction(toolName: string, rawArgs: unknown): string { + const a = (rawArgs ?? {}) as Record; + switch (toolName) { + case 'computer_batch': + return Array.isArray(a.actions) + ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') + : 'batch'; + case 'computer_use_extra': + // OpenAI's navigation helper wraps the real action under `action`. + return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); + case 'screenshot': + return 'screenshot'; + case 'goto': + return `goto ${a.url ?? ''}`.trim(); + case 'click': + case 'left_click': + case 'double_click': + case 'right_click': + return `${toolName} ${point(a)}`; + case 'drag': + return `drag ${dragPath(a.path)}`; + case 'keypress': + return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; + case 'type': + return `type ${quote(a.text)}`; + case 'scroll': + return `scroll ${point(a)}`.trim(); + case 'wait': + return `wait ${a.ms ?? ''}ms`; + default: + return `${toolName ?? 'action'} ${compact(a)}`.trim(); } } -function formatJson(value: unknown): string { - const text = JSON.stringify(value) ?? 'undefined'; - return text.length > 500 ? `${text.slice(0, 497)}...` : text; +function point(a: Record): string { + if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; + if (typeof a.x === 'number') return `(${a.x})`; + return ''; +} + +function dragPath(path: unknown): string { + if (!Array.isArray(path) || path.length === 0) return ''; + const first = path[0]; + const last = path[path.length - 1]; + return `${point(first)} → ${point(last)}`; +} + +function quote(text: unknown): string { + const s = typeof text === 'string' ? text : String(text ?? ''); + return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +} + +function compact(value: unknown): string { + const text = JSON.stringify(value) ?? ''; + return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; } diff --git a/pkg/templates/typescript/openai-computer-use/lib/logging.ts b/pkg/templates/typescript/openai-computer-use/lib/logging.ts index 2090e0d2..d1f5ad93 100644 --- a/pkg/templates/typescript/openai-computer-use/lib/logging.ts +++ b/pkg/templates/typescript/openai-computer-use/lib/logging.ts @@ -1,22 +1,99 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Logs each computer-use tool call as CuaAgent executes it. Pass to -// `agent.subscribe(logAgentEvent)` to see every action (click, type, -// screenshot, ...) as it happens instead of only the final answer. +// Human-readable trace of what the agent is doing, meant for +// `agent.subscribe(logAgentEvent)`: the model's narration between steps +// (`agent>`), then each concrete browser action it takes (`→`). Failed +// actions are marked so retries are visible. export function logAgentEvent(event: AgentEvent): void { - if (event.type === 'tool_execution_start') { - console.log(`[tool:start] ${event.toolName} args=${formatJson(event.args)}`); - return; + switch (event.type) { + case 'message_end': { + const text = assistantText(event.message); + if (text) console.log(`\nagent> ${text}`); + return; + } + case 'tool_execution_start': + console.log(` → ${formatAction(event.toolName, event.args)}`); + return; + case 'tool_execution_end': + if (event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } + return; } - if (event.type === 'tool_execution_end') { - // Log the structured `details` rather than the full result: `result.content` - // is what's sent back to the model and includes screenshot image bytes. - const details = (event.result as { details?: unknown } | undefined)?.details; - console.log(`[tool:end] ${event.toolName} error=${event.isError} details=${formatJson(details)}`); +} + +// Concatenate the visible text blocks of an assistant message. Non-assistant +// messages and tool-call-only turns have no narration and return ''. +function assistantText(message: unknown): string { + const m = message as { role?: string; content?: unknown }; + if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; + return m.content + .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) + .map((b) => b.text.trim()) + .filter(Boolean) + .join(' '); +} + +function isTextBlock(b: unknown): boolean { + return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; +} + +// Render one tool call as a short action line. Anthropic batches actions under +// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow +// through here — batch sub-actions carry their kind in `type`. +function formatAction(toolName: string, rawArgs: unknown): string { + const a = (rawArgs ?? {}) as Record; + switch (toolName) { + case 'computer_batch': + return Array.isArray(a.actions) + ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') + : 'batch'; + case 'computer_use_extra': + // OpenAI's navigation helper wraps the real action under `action`. + return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); + case 'screenshot': + return 'screenshot'; + case 'goto': + return `goto ${a.url ?? ''}`.trim(); + case 'click': + case 'left_click': + case 'double_click': + case 'right_click': + return `${toolName} ${point(a)}`; + case 'drag': + return `drag ${dragPath(a.path)}`; + case 'keypress': + return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; + case 'type': + return `type ${quote(a.text)}`; + case 'scroll': + return `scroll ${point(a)}`.trim(); + case 'wait': + return `wait ${a.ms ?? ''}ms`; + default: + return `${toolName ?? 'action'} ${compact(a)}`.trim(); } } -function formatJson(value: unknown): string { - const text = JSON.stringify(value) ?? 'undefined'; - return text.length > 500 ? `${text.slice(0, 497)}...` : text; +function point(a: Record): string { + if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; + if (typeof a.x === 'number') return `(${a.x})`; + return ''; +} + +function dragPath(path: unknown): string { + if (!Array.isArray(path) || path.length === 0) return ''; + const first = path[0]; + const last = path[path.length - 1]; + return `${point(first)} → ${point(last)}`; +} + +function quote(text: unknown): string { + const s = typeof text === 'string' ? text : String(text ?? ''); + return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +} + +function compact(value: unknown): string { + const text = JSON.stringify(value) ?? ''; + return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; } From a489ac74f3f920cf6ff5dc1ad113c1cce627fc3c Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:22:43 +0000 Subject: [PATCH 4/4] Simplify the action logger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-tool formatters (point/dragPath/quote/keypress/etc.) with a generic key=value renderer, keeping only the two unwraps that materially help readability: Anthropic's computer_batch and OpenAI's computer_use_extra. Collapse narration whitespace so each agent> line stays on one line. ~99 → 45 lines, same clean output across all three providers. --- .../anthropic-computer-use/logging.ts | 120 +++++------------- .../typescript/gemini-computer-use/logging.ts | 120 +++++------------- .../openai-computer-use/lib/logging.ts | 120 +++++------------- 3 files changed, 99 insertions(+), 261 deletions(-) diff --git a/pkg/templates/typescript/anthropic-computer-use/logging.ts b/pkg/templates/typescript/anthropic-computer-use/logging.ts index d1f5ad93..31f52dae 100644 --- a/pkg/templates/typescript/anthropic-computer-use/logging.ts +++ b/pkg/templates/typescript/anthropic-computer-use/logging.ts @@ -1,99 +1,45 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Human-readable trace of what the agent is doing, meant for -// `agent.subscribe(logAgentEvent)`: the model's narration between steps -// (`agent>`), then each concrete browser action it takes (`→`). Failed -// actions are marked so retries are visible. +// Logs the agent's narration (`agent>`) and each browser action (`→`) it takes. +// Pass to `agent.subscribe(logAgentEvent)`. export function logAgentEvent(event: AgentEvent): void { - switch (event.type) { - case 'message_end': { - const text = assistantText(event.message); - if (text) console.log(`\nagent> ${text}`); - return; - } - case 'tool_execution_start': - console.log(` → ${formatAction(event.toolName, event.args)}`); - return; - case 'tool_execution_end': - if (event.isError) { - console.log(` ✗ ${event.toolName} failed`); - } - return; + if (event.type === 'tool_execution_start') { + console.log(` → ${describe(event.toolName, event.args)}`); + } else if (event.type === 'tool_execution_end' && event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } else if (event.type === 'message_end') { + const text = narration(event.message); + if (text) console.log(`\nagent> ${text}`); } } -// Concatenate the visible text blocks of an assistant message. Non-assistant -// messages and tool-call-only turns have no narration and return ''. -function assistantText(message: unknown): string { - const m = message as { role?: string; content?: unknown }; - if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; - return m.content - .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) - .map((b) => b.text.trim()) - .filter(Boolean) - .join(' '); -} - -function isTextBlock(b: unknown): boolean { - return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; -} - -// Render one tool call as a short action line. Anthropic batches actions under -// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow -// through here — batch sub-actions carry their kind in `type`. -function formatAction(toolName: string, rawArgs: unknown): string { - const a = (rawArgs ?? {}) as Record; - switch (toolName) { - case 'computer_batch': - return Array.isArray(a.actions) - ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') - : 'batch'; - case 'computer_use_extra': - // OpenAI's navigation helper wraps the real action under `action`. - return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); - case 'screenshot': - return 'screenshot'; - case 'goto': - return `goto ${a.url ?? ''}`.trim(); - case 'click': - case 'left_click': - case 'double_click': - case 'right_click': - return `${toolName} ${point(a)}`; - case 'drag': - return `drag ${dragPath(a.path)}`; - case 'keypress': - return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; - case 'type': - return `type ${quote(a.text)}`; - case 'scroll': - return `scroll ${point(a)}`.trim(); - case 'wait': - return `wait ${a.ms ?? ''}ms`; - default: - return `${toolName ?? 'action'} ${compact(a)}`.trim(); +// Anthropic nests actions under `computer_batch`, OpenAI under +// `computer_use_extra`; unwrap those, then print the action name and its args. +function describe(name: string, args: any): string { + if (name === 'computer_batch' && Array.isArray(args?.actions)) { + return args.actions.map((a: any) => describe(a.type, a)).join('; '); } + if (name === 'computer_use_extra' && typeof args?.action === 'string') { + return describe(args.action, args); + } + const params = Object.entries(args ?? {}) + .filter(([key]) => key !== 'type' && key !== 'action') + .map(([key, value]) => `${key}=${short(value)}`) + .join(' '); + return params ? `${name} ${params}` : name; } -function point(a: Record): string { - if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; - if (typeof a.x === 'number') return `(${a.x})`; - return ''; -} - -function dragPath(path: unknown): string { - if (!Array.isArray(path) || path.length === 0) return ''; - const first = path[0]; - const last = path[path.length - 1]; - return `${point(first)} → ${point(last)}`; -} - -function quote(text: unknown): string { - const s = typeof text === 'string' ? text : String(text ?? ''); - return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +function narration(message: any): string { + if (message?.role !== 'assistant') return ''; + return message.content + .filter((block: any) => block.type === 'text') + .map((block: any) => block.text) + .join(' ') + .replace(/\s+/g, ' ') + .trim(); } -function compact(value: unknown): string { - const text = JSON.stringify(value) ?? ''; - return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; +function short(value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value); + return text.length > 80 ? `${text.slice(0, 77)}...` : text; } diff --git a/pkg/templates/typescript/gemini-computer-use/logging.ts b/pkg/templates/typescript/gemini-computer-use/logging.ts index d1f5ad93..31f52dae 100644 --- a/pkg/templates/typescript/gemini-computer-use/logging.ts +++ b/pkg/templates/typescript/gemini-computer-use/logging.ts @@ -1,99 +1,45 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Human-readable trace of what the agent is doing, meant for -// `agent.subscribe(logAgentEvent)`: the model's narration between steps -// (`agent>`), then each concrete browser action it takes (`→`). Failed -// actions are marked so retries are visible. +// Logs the agent's narration (`agent>`) and each browser action (`→`) it takes. +// Pass to `agent.subscribe(logAgentEvent)`. export function logAgentEvent(event: AgentEvent): void { - switch (event.type) { - case 'message_end': { - const text = assistantText(event.message); - if (text) console.log(`\nagent> ${text}`); - return; - } - case 'tool_execution_start': - console.log(` → ${formatAction(event.toolName, event.args)}`); - return; - case 'tool_execution_end': - if (event.isError) { - console.log(` ✗ ${event.toolName} failed`); - } - return; + if (event.type === 'tool_execution_start') { + console.log(` → ${describe(event.toolName, event.args)}`); + } else if (event.type === 'tool_execution_end' && event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } else if (event.type === 'message_end') { + const text = narration(event.message); + if (text) console.log(`\nagent> ${text}`); } } -// Concatenate the visible text blocks of an assistant message. Non-assistant -// messages and tool-call-only turns have no narration and return ''. -function assistantText(message: unknown): string { - const m = message as { role?: string; content?: unknown }; - if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; - return m.content - .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) - .map((b) => b.text.trim()) - .filter(Boolean) - .join(' '); -} - -function isTextBlock(b: unknown): boolean { - return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; -} - -// Render one tool call as a short action line. Anthropic batches actions under -// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow -// through here — batch sub-actions carry their kind in `type`. -function formatAction(toolName: string, rawArgs: unknown): string { - const a = (rawArgs ?? {}) as Record; - switch (toolName) { - case 'computer_batch': - return Array.isArray(a.actions) - ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') - : 'batch'; - case 'computer_use_extra': - // OpenAI's navigation helper wraps the real action under `action`. - return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); - case 'screenshot': - return 'screenshot'; - case 'goto': - return `goto ${a.url ?? ''}`.trim(); - case 'click': - case 'left_click': - case 'double_click': - case 'right_click': - return `${toolName} ${point(a)}`; - case 'drag': - return `drag ${dragPath(a.path)}`; - case 'keypress': - return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; - case 'type': - return `type ${quote(a.text)}`; - case 'scroll': - return `scroll ${point(a)}`.trim(); - case 'wait': - return `wait ${a.ms ?? ''}ms`; - default: - return `${toolName ?? 'action'} ${compact(a)}`.trim(); +// Anthropic nests actions under `computer_batch`, OpenAI under +// `computer_use_extra`; unwrap those, then print the action name and its args. +function describe(name: string, args: any): string { + if (name === 'computer_batch' && Array.isArray(args?.actions)) { + return args.actions.map((a: any) => describe(a.type, a)).join('; '); } + if (name === 'computer_use_extra' && typeof args?.action === 'string') { + return describe(args.action, args); + } + const params = Object.entries(args ?? {}) + .filter(([key]) => key !== 'type' && key !== 'action') + .map(([key, value]) => `${key}=${short(value)}`) + .join(' '); + return params ? `${name} ${params}` : name; } -function point(a: Record): string { - if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; - if (typeof a.x === 'number') return `(${a.x})`; - return ''; -} - -function dragPath(path: unknown): string { - if (!Array.isArray(path) || path.length === 0) return ''; - const first = path[0]; - const last = path[path.length - 1]; - return `${point(first)} → ${point(last)}`; -} - -function quote(text: unknown): string { - const s = typeof text === 'string' ? text : String(text ?? ''); - return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +function narration(message: any): string { + if (message?.role !== 'assistant') return ''; + return message.content + .filter((block: any) => block.type === 'text') + .map((block: any) => block.text) + .join(' ') + .replace(/\s+/g, ' ') + .trim(); } -function compact(value: unknown): string { - const text = JSON.stringify(value) ?? ''; - return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; +function short(value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value); + return text.length > 80 ? `${text.slice(0, 77)}...` : text; } diff --git a/pkg/templates/typescript/openai-computer-use/lib/logging.ts b/pkg/templates/typescript/openai-computer-use/lib/logging.ts index d1f5ad93..31f52dae 100644 --- a/pkg/templates/typescript/openai-computer-use/lib/logging.ts +++ b/pkg/templates/typescript/openai-computer-use/lib/logging.ts @@ -1,99 +1,45 @@ import type { AgentEvent } from '@onkernel/cua-agent'; -// Human-readable trace of what the agent is doing, meant for -// `agent.subscribe(logAgentEvent)`: the model's narration between steps -// (`agent>`), then each concrete browser action it takes (`→`). Failed -// actions are marked so retries are visible. +// Logs the agent's narration (`agent>`) and each browser action (`→`) it takes. +// Pass to `agent.subscribe(logAgentEvent)`. export function logAgentEvent(event: AgentEvent): void { - switch (event.type) { - case 'message_end': { - const text = assistantText(event.message); - if (text) console.log(`\nagent> ${text}`); - return; - } - case 'tool_execution_start': - console.log(` → ${formatAction(event.toolName, event.args)}`); - return; - case 'tool_execution_end': - if (event.isError) { - console.log(` ✗ ${event.toolName} failed`); - } - return; + if (event.type === 'tool_execution_start') { + console.log(` → ${describe(event.toolName, event.args)}`); + } else if (event.type === 'tool_execution_end' && event.isError) { + console.log(` ✗ ${event.toolName} failed`); + } else if (event.type === 'message_end') { + const text = narration(event.message); + if (text) console.log(`\nagent> ${text}`); } } -// Concatenate the visible text blocks of an assistant message. Non-assistant -// messages and tool-call-only turns have no narration and return ''. -function assistantText(message: unknown): string { - const m = message as { role?: string; content?: unknown }; - if (m.role !== 'assistant' || !Array.isArray(m.content)) return ''; - return m.content - .filter((b): b is { type: 'text'; text: string } => isTextBlock(b)) - .map((b) => b.text.trim()) - .filter(Boolean) - .join(' '); -} - -function isTextBlock(b: unknown): boolean { - return typeof b === 'object' && b !== null && (b as { type?: string }).type === 'text'; -} - -// Render one tool call as a short action line. Anthropic batches actions under -// `computer_batch`; OpenAI/Gemini emit `goto`/`click`/... directly. Both flow -// through here — batch sub-actions carry their kind in `type`. -function formatAction(toolName: string, rawArgs: unknown): string { - const a = (rawArgs ?? {}) as Record; - switch (toolName) { - case 'computer_batch': - return Array.isArray(a.actions) - ? a.actions.map((sub: any) => formatAction(sub?.type, sub)).join('; ') - : 'batch'; - case 'computer_use_extra': - // OpenAI's navigation helper wraps the real action under `action`. - return typeof a.action === 'string' ? formatAction(a.action, a) : compact(a); - case 'screenshot': - return 'screenshot'; - case 'goto': - return `goto ${a.url ?? ''}`.trim(); - case 'click': - case 'left_click': - case 'double_click': - case 'right_click': - return `${toolName} ${point(a)}`; - case 'drag': - return `drag ${dragPath(a.path)}`; - case 'keypress': - return `keypress ${Array.isArray(a.keys) ? a.keys.join('+') : compact(a)}`; - case 'type': - return `type ${quote(a.text)}`; - case 'scroll': - return `scroll ${point(a)}`.trim(); - case 'wait': - return `wait ${a.ms ?? ''}ms`; - default: - return `${toolName ?? 'action'} ${compact(a)}`.trim(); +// Anthropic nests actions under `computer_batch`, OpenAI under +// `computer_use_extra`; unwrap those, then print the action name and its args. +function describe(name: string, args: any): string { + if (name === 'computer_batch' && Array.isArray(args?.actions)) { + return args.actions.map((a: any) => describe(a.type, a)).join('; '); } + if (name === 'computer_use_extra' && typeof args?.action === 'string') { + return describe(args.action, args); + } + const params = Object.entries(args ?? {}) + .filter(([key]) => key !== 'type' && key !== 'action') + .map(([key, value]) => `${key}=${short(value)}`) + .join(' '); + return params ? `${name} ${params}` : name; } -function point(a: Record): string { - if (typeof a.x === 'number' && typeof a.y === 'number') return `(${a.x}, ${a.y})`; - if (typeof a.x === 'number') return `(${a.x})`; - return ''; -} - -function dragPath(path: unknown): string { - if (!Array.isArray(path) || path.length === 0) return ''; - const first = path[0]; - const last = path[path.length - 1]; - return `${point(first)} → ${point(last)}`; -} - -function quote(text: unknown): string { - const s = typeof text === 'string' ? text : String(text ?? ''); - return `"${s.length > 80 ? `${s.slice(0, 77)}...` : s}"`; +function narration(message: any): string { + if (message?.role !== 'assistant') return ''; + return message.content + .filter((block: any) => block.type === 'text') + .map((block: any) => block.text) + .join(' ') + .replace(/\s+/g, ' ') + .trim(); } -function compact(value: unknown): string { - const text = JSON.stringify(value) ?? ''; - return text === '{}' ? '' : text.length > 120 ? `${text.slice(0, 117)}...` : text; +function short(value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value); + return text.length > 80 ? `${text.slice(0, 77)}...` : text; }