From e2b3ba23be1e87cae74d4fd4bbe68e7661122177 Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:28:50 +0000 Subject: [PATCH 1/2] templates: upgrade Yutori templates from n1.5 to n2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit n2 is a computer-use model, not a browser-use model, so this is more than a model id swap. The API surface changed in ways that break the n1.5 loop: - GUI actions now arrive as a single `computer_batch` call carrying an action list instead of one tool call per action. The loop unpacks the batch, runs it in order, stops at the first error, and answers with one tool result and one screenshot taken after the last action that ran. - `disable_tools` is rejected with a 400, so `bash`, `read`, `write`, and `edit` are always served. New system tool module backs them with Kernel's process and filesystem APIs. - `goto_url`, `go_back`, `go_forward`, `refresh`, and horizontal scroll are gone — n2 drives the address bar and browser buttons directly. Kiosk mode is removed with them: it hides the address bar n2 needs, and n2 wants a whole-screen capture rather than a cropped one. - `scroll` `amount` is now a wheel notch count, matching Kernel's delta_y 1:1. - `mouse_down` / `mouse_up` take optional coordinates for manual drags. - Key space gained punctuation word forms. - `reasoning_content` is echoed back on each turn, `reasoning_effort` is exposed on the payload, and `max_completion_tokens` is raised to 16384 since the reasoning trace shares that budget. - Screenshot trimming now matches n2's server-side retention: images are kept only on the last 2 image-bearing messages, with the 10 MB cap as a backstop. - `prev_request_id` threads through so a trajectory groups into one conversation for usage reporting. Verified end-to-end against the live n2 API and a Kernel browser in both languages, including a `bash` call executed on the browser VM. Co-Authored-By: Claude Opus 5 --- pkg/create/templates.go | 4 +- pkg/templates/python/yutori/README.md | 90 ++-- pkg/templates/python/yutori/loop.py | 341 ++++++------ pkg/templates/python/yutori/main.py | 23 +- pkg/templates/python/yutori/pyproject.toml | 2 +- pkg/templates/python/yutori/session.py | 4 - pkg/templates/python/yutori/tools/__init__.py | 12 +- pkg/templates/python/yutori/tools/base.py | 10 +- pkg/templates/python/yutori/tools/computer.py | 486 ++++++++---------- pkg/templates/python/yutori/tools/system.py | 184 +++++++ pkg/templates/typescript/yutori/README.md | 90 ++-- pkg/templates/typescript/yutori/index.ts | 18 +- pkg/templates/typescript/yutori/loop.ts | 373 ++++++-------- pkg/templates/typescript/yutori/session.ts | 8 - .../typescript/yutori/tools/computer.ts | 464 +++++++---------- .../typescript/yutori/tools/system.ts | 225 ++++++++ 16 files changed, 1237 insertions(+), 1097 deletions(-) create mode 100644 pkg/templates/python/yutori/tools/system.py create mode 100644 pkg/templates/typescript/yutori/tools/system.ts diff --git a/pkg/create/templates.go b/pkg/create/templates.go index 0627ad3c..b14c08d2 100644 --- a/pkg/create/templates.go +++ b/pkg/create/templates.go @@ -87,8 +87,8 @@ var Templates = map[string]TemplateInfo{ Languages: []string{LanguageTypeScript, LanguagePython}, }, TemplateYutoriComputerUse: { - Name: "Yutori n1.5 Computer Use", - Description: "Implements a Yutori n1.5 computer use agent", + Name: "Yutori n2 Computer Use", + Description: "Implements a Yutori n2 computer use agent", Languages: []string{LanguageTypeScript, LanguagePython}, }, TemplateTzafonComputerUse: { diff --git a/pkg/templates/python/yutori/README.md b/pkg/templates/python/yutori/README.md index afca2702..16fe66b6 100644 --- a/pkg/templates/python/yutori/README.md +++ b/pkg/templates/python/yutori/README.md @@ -1,10 +1,8 @@ -# Kernel Python Sample App - Yutori n1.5 Computer Use +# Kernel Python Sample App - Yutori n2 Computer Use -This Kernel app implements a prompt loop using Yutori's Navigator n1.5 with Kernel's Computer Controls API. +This Kernel app implements a prompt loop using Yutori's Navigator n2 with Kernel's Computer Controls API. -[Navigator n1.5](https://yutori.com/blog/introducing-n1-5) is Yutori's pixels-to-actions LLM that predicts browser actions from screenshots. - -This template runs n1.5 in **computer-use-only mode**. n1.5 also supports a hybrid vision + DOM/JavaScript path (page-state extraction, custom JS, structured JSON output) for multi-field forms and bulk data extraction, but those tools are intentionally disabled here — see [Disabled tools](#disabled-tools). +[Navigator n2](https://docs.yutori.com/reference/n2) is Yutori's computer-use model. It reads a screenshot of the whole screen and answers with a batch of mouse and keyboard actions, and it can run shell commands and edit files on the machine it is driving. ## Setup @@ -28,7 +26,7 @@ kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https Optional payload fields: - `record_replay` (bool) — capture a video of the session (paid plans only). -- `kiosk` (bool) — launch the browser without address bar / tabs ([see below](#kiosk-mode)). +- `reasoning_effort` (`"none"`, `"low"`, `"medium"`, `"xhigh"`) — n2 reasons at `medium` by default. `xhigh` gives the longest traces and does best on hard multi-step tasks; `none` turns reasoning off. - `user_timezone` (IANA, e.g. `"America/New_York"`) and `user_location` (free text, e.g. `"New York, NY, US"`) — appended to the task message so the model has accurate temporal/locational grounding. More involved example (Kanban drag-and-drop): @@ -49,66 +47,62 @@ kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https When enabled, the response will include a `replay_url` field with a link to view the recorded session. -## Kiosk mode - -Prefer **non-kiosk mode** by default and when the agent is expected to switch domains via URL. Use **kiosk (`"kiosk": true`)** when: (1) you're recording sessions and want a cleaner UI in the replay, or (2) you're automating on a single website and the combination of the complex site layout and browser chrome (address bar, tabs) may confuse the agent. - -Note: In kiosk mode the agent may still try to use the address bar to enter URLs; it's not available, so it will eventually use `goto_url`, but those attempts may result in slowdown of the overall session. +## Screen Configuration -Default (non-kiosk): - -```bash -kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com, then navigate to ign.com and describe the page"}' -``` +n2 is a desktop model: it expects a screenshot of the **whole screen**, browser chrome included, and it navigates by clicking the address bar rather than through a dedicated navigation action. A Kernel `viewport` sets the browser window size, and screenshots from Computer Controls capture that window with its tabs and toolbar — which is exactly what n2 wants, so this template does not use kiosk mode. -With kiosk (single-site or recording): +This template runs at **1280x800**. Yutori also lists 1920x1080 and 1280x720 as resolutions in regular use; grounding may degrade at extreme aspect ratios. -```bash -kernel invoke python-yutori-cua cua-task --payload '{"query": "Enter https://example.com in the search box and then describe the page.", "kiosk": true}' -``` +> **Note:** n2 outputs coordinates in a 1000x1000 relative space, which are scaled to the actual screen dimensions per action. -## Viewport Configuration +See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. -Yutori n1.5 recommends a **1280×800 (WXGA, 16:10)** viewport for best grounding accuracy. +## Screenshots -> **Note:** n1.5 outputs coordinates in a 1000×1000 relative space, which are automatically scaled to the actual viewport dimensions. +Screenshots are converted to WebP before they are sent. Yutori caps requests at 10 MB, and a full-screen PNG trajectory blows past that on its own. -See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. +The loop also drops screenshots the model will not read: n2 keeps images from only the last 2 image-bearing messages, so older ones are stripped from each request while every message's text is kept. -## Screenshots +## Tools -Screenshots are automatically converted to WebP format for better compression across multi-step trajectories, as recommended by Yutori. +This template pins the `computer_use_tools-20260825` tool set. n2 rejects `disable_tools`, so all five tools are always served and the loop answers all of them. -## n1.5-latest Supported Actions +### `computer_batch` -This template uses the `browser_tools_core-20260403` tool set — coordinate-based browser actions that operate on screenshots only. +n2 returns a whole action list in one call. The loop runs them in order, stops at the first error, and replies with a single tool result carrying one screenshot taken after the last action that ran. Every coordinate in a batch refers to the screenshot from *before* the batch started. | Action | Description | |--------|-------------| | `left_click` | Left mouse click at coordinates (supports `modifier`) | | `double_click` | Double-click at coordinates (supports `modifier`) | | `triple_click` | Triple-click at coordinates (supports `modifier`) | -| `middle_click` | Middle mouse click at coordinates | -| `right_click` | Right mouse click at coordinates | -| `mouse_move` | Move mouse to coordinates without clicking | -| `mouse_down` | Press the left mouse button at coordinates | -| `mouse_up` | Release the left mouse button at coordinates | -| `scroll` | Scroll page in a direction | -| `type` | Type text into focused element | -| `key_press` | Send a single key or key combination | -| `hold_key` | Hold a key for a duration | -| `drag` | Click-and-drag operation | -| `wait` | Pause for UI to update | -| `refresh` | Reload current page | -| `go_back` | Navigate back in history | -| `go_forward` | Navigate forward in history | -| `goto_url` | Navigate to a URL | - -### Disabled tools - -The DOM/Playwright-based "expanded" tools (`extract_elements`, `find`, `set_element_value`, `execute_js`) are intentionally disabled via the `disable_tools` request parameter — this template runs computer-use only and does not expose a Playwright page to the model. +| `middle_click` | Middle mouse click at coordinates (supports `modifier`) | +| `right_click` | Right mouse click at coordinates (supports `modifier`) | +| `scroll` | Scroll up or down at coordinates, in wheel notches | +| `type` | Type text into the focused element | +| `key_press` | Press a key, combination, or sequence | +| `drag` | Drag from `start_coordinates` to `coordinates` | +| `mouse_move` | Move the mouse to coordinates without clicking | +| `mouse_down` | Press and hold the left button (coordinates optional) | +| `mouse_up` | Release the left button (coordinates optional) | +| `hold_key` | Hold a key down for a duration | +| `wait` | Pause without interacting | +| `screenshot` | Look without acting | + +Horizontal scrolling is not available, and there are no `goto_url` / `go_back` / `go_forward` / `refresh` actions — n2 drives the address bar and browser buttons like a person would. + +### `bash`, `read`, `write`, `edit` + +n2 can also work on the browser VM directly. These map onto Kernel's process and filesystem APIs: + +| Tool | Backed by | +|------|-----------| +| `bash` | `browsers.process.exec` — each call is a separate process; the working directory carries over between calls, environment variables do not. `run_in_background` detaches the command and reports its pid and log path. | +| `read` | `browsers.fs.read_file` — returns `cat -n` output, with `offset` / `limit` paging | +| `write` | `browsers.fs.write_file` | +| `edit` | read, exact-string replace, write. Refuses to edit a file that has not been read in this session. | ## Resources -- [Yutori n1.5 API Documentation](https://docs.yutori.com/reference/n1-5) +- [Yutori n2 API Documentation](https://docs.yutori.com/reference/n2) - [Kernel Documentation](https://www.kernel.sh/docs/quickstart) diff --git a/pkg/templates/python/yutori/loop.py b/pkg/templates/python/yutori/loop.py index 1a81c491..ead901b4 100644 --- a/pkg/templates/python/yutori/loop.py +++ b/pkg/templates/python/yutori/loop.py @@ -1,14 +1,15 @@ """ -Yutori n1.5 Sampling Loop +Yutori n2 Sampling Loop -Implements the agent loop for Yutori's n1.5-latest computer use model. -n1.5-latest uses an OpenAI-compatible API with tool_calls: -- Actions are returned via tool_calls in the assistant message -- Tool results use role: "tool" with matching tool_call_id +Implements the agent loop for Yutori's n2 computer use model. +n2 uses an OpenAI-compatible API with tool_calls: +- GUI actions arrive as a single `computer_batch` call holding an action list +- `bash`, `read`, `write`, and `edit` arrive as their own calls +- Every call needs one tool result with a matching tool_call_id - The model stops by returning content without tool_calls - Coordinates are returned in 1000x1000 space and need scaling -@see https://docs.yutori.com/reference/n1-5 +@see https://docs.yutori.com/reference/n2 """ from __future__ import annotations @@ -23,107 +24,96 @@ from kernel import Kernel from openai import OpenAI -from tools import ComputerTool, N15Action, ToolResult +from tools import ComputerTool, SystemTools -# Tools that require a Playwright page / DOM access. The default core tool set -# already excludes them, but we also list them in `disable_tools` so the -# exclusion is explicit and survives if the default ever changes. -DISABLED_TOOLS = ["extract_elements", "find", "set_element_value", "execute_js"] -TOOL_SET = "browser_tools_core-20260403" +# Pin the tool set so a change to the server default can't change which tools +# this run exposes. n2 rejects `disable_tools`, so the set is served whole. +TOOL_SET = "computer_use_tools-20260825" -NAVIGATOR_COORDINATE_SCALE = 1000 - -# Screenshot-trimming defaults mirror Yutori's reference loop: -# https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/payload.py -# Trimming is size-triggered — we only drop old screenshots when the payload -# exceeds MAX_REQUEST_BYTES, and we always keep at least KEEP_RECENT_SCREENSHOTS. +# Requests are capped at 10 MB. n2 also only reads images from the last 2 +# image-bearing messages, so anything older is dropped before sending rather +# than paying to upload screenshots the model will discard server-side. MAX_REQUEST_BYTES = 9_500_000 -KEEP_RECENT_SCREENSHOTS = 6 +KEEP_IMAGE_MESSAGES = 2 async def sampling_loop( *, - model: str = "n1.5-latest", + model: str = "n2", task: str, api_key: str, kernel: Kernel, session_id: str, - max_completion_tokens: int = 4096, + # Shared by the reasoning trace and the tool call, so a low value truncates + # the call. + max_completion_tokens: int = 16384, max_iterations: int = 100, - viewport_width: int = 1280, - viewport_height: int = 800, - kiosk_mode: bool = False, + screen_width: int = 1280, + screen_height: int = 800, + reasoning_effort: Optional[str] = None, user_timezone: str = "America/Los_Angeles", user_location: str = "San Francisco, CA, US", ) -> dict[str, Any]: - """Run the n1.5 sampling loop until the model stops calling tools or max iterations.""" - client = OpenAI( - api_key=api_key, - base_url="https://api.yutori.com/v1", - ) - - computer_tool = ComputerTool(kernel, session_id, viewport_width, viewport_height, kiosk_mode=kiosk_mode) + """Run the n2 sampling loop until the model stops calling tools or max iterations.""" + client = OpenAI(api_key=api_key, base_url="https://api.yutori.com/v1") - initial_screenshot = await computer_tool.screenshot() + computer_tool = ComputerTool(kernel, session_id, screen_width, screen_height) + system_tools = SystemTools(kernel, session_id) # Append location/timezone/current-date context to the task — mirrors Yutori's # format_task_with_context helper and helps the model with date-sensitive # judgments. https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/context.py - task_with_context = _format_task_with_context(task, user_timezone, user_location) - - user_content: list[dict[str, Any]] = [{"type": "text", "text": task_with_context}] - if initial_screenshot.get("base64_image"): - user_content.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{initial_screenshot['base64_image']}" - }, - }) - conversation_messages: list[dict[str, Any]] = [ - {"role": "user", "content": user_content} + { + "role": "user", + "content": [ + {"type": "text", "text": _format_task_with_context(task, user_timezone, user_location)}, + _image_part(await computer_tool.screenshot()), + ], + } ] iteration = 0 final_answer: Optional[str] = None + prev_request_id: Optional[str] = None while iteration < max_iterations: iteration += 1 print(f"\n=== Iteration {iteration} ===") - request_messages, dropped = _trimmed_for_request(conversation_messages) - if dropped: - print(f"Trimmed {dropped} old screenshot(s) to fit request size limit") + request_messages = _trimmed_for_request(conversation_messages) + + # n2-specific knobs go in extra_body. + extra_body: dict[str, Any] = {"tool_set": TOOL_SET} + if reasoning_effort: + extra_body["reasoning_effort"] = reasoning_effort + if prev_request_id: + # Groups the trajectory into one conversation for usage reporting. + extra_body["prev_request_id"] = prev_request_id try: response = client.chat.completions.create( model=model, messages=request_messages, max_completion_tokens=max_completion_tokens, - temperature=0.3, - # n1.5-specific knobs go in extra_body. - # tool_set selects the core (coordinate-based) tools. - # disable_tools is a defense-in-depth exclusion of DOM/Playwright tools. - extra_body={ - "tool_set": TOOL_SET, - "disable_tools": DISABLED_TOOLS, - }, + temperature=0.6, + extra_body=extra_body, ) except Exception as api_error: print(f"API call failed: {api_error}") raise - if not response.choices or len(response.choices) == 0: - print(f"No choices in response: {response}") - raise ValueError("No choices in API response") + prev_request_id = getattr(response, "_request_id", None) or prev_request_id - choice = response.choices[0] - assistant_message = choice.message - if not assistant_message: + if not response.choices: + print(f"No choices in response: {response}") raise ValueError("No response from model") + assistant_message = response.choices[0].message print("Assistant content:", assistant_message.content or "(none)") + # Push the assistant message unchanged — `reasoning_content` rides along + # on it, and n2 needs it echoed back to keep its reasoning across turns. conversation_messages.append(assistant_message.model_dump(exclude_none=True)) tool_calls = assistant_message.tool_calls @@ -134,56 +124,25 @@ async def sampling_loop( print(f"No tool_calls, model is done. Final answer: {final_answer}") break - for tc in tool_calls: - action_name = tc.function.name - try: - args = json.loads(tc.function.arguments) - except json.JSONDecodeError: - print(f"Failed to parse tool_call arguments: {tc.function.arguments}") + for index, tool_call in enumerate(tool_calls): + # n2 usually answers with one call, but `parallel_tool_calls` is + # pinned on server-side so a turn can carry several. Only the first + # one was planned against the screenshot the model actually saw, so + # run it and make the rest re-plan. + if index > 0: conversation_messages.append({ "role": "tool", - "tool_call_id": tc.id, - "content": "Error: failed to parse arguments", + "tool_call_id": tool_call.id, + "content": ( + "Not executed: planned against a screenshot the previous call already " + "changed. Re-plan from the latest screenshot." + ), }) continue - action: N15Action = {"action_type": action_name, **args} - print(f"Executing action: {action_name}", args) - - scaled_action = _scale_coordinates(action, viewport_width, viewport_height) - - result: ToolResult - try: - result = await computer_tool.execute(scaled_action) - except Exception as e: - print(f"Action failed: {e}") - result = {"error": str(e)} - - if result.get("base64_image"): - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{result['base64_image']}" - }, - } - ], - }) - elif result.get("error"): - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": f"Action failed: {result['error']}", - }) - else: - conversation_messages.append({ - "role": "tool", - "tool_call_id": tc.id, - "content": result.get("output", "OK"), - }) + conversation_messages.append( + await _run_tool_call(tool_call, computer_tool, system_tools) + ) # If the loop exhausted iterations, prompt the model for a final summary so # the caller gets a usable answer instead of empty content. Mirrors Yutori's @@ -191,29 +150,23 @@ async def sampling_loop( if iteration >= max_iterations and not final_answer: print("Max iterations reached — requesting summary") try: - final_screenshot = await computer_tool.screenshot() - stop_content: list[dict[str, Any]] = [ - {"type": "text", "text": _format_stop_and_summarize(task)} - ] - if final_screenshot.get("base64_image"): - stop_content.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/webp;base64,{final_screenshot['base64_image']}" - }, - }) - conversation_messages.append({"role": "user", "content": stop_content}) + conversation_messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": _format_stop_and_summarize(task)}, + _image_part(await computer_tool.screenshot()), + ], + }) - summary_messages, _ = _trimmed_for_request(conversation_messages) summary_response = client.chat.completions.create( model=model, - messages=summary_messages, + messages=_trimmed_for_request(conversation_messages), max_completion_tokens=max_completion_tokens, - temperature=0.3, - extra_body={"tool_set": TOOL_SET, "disable_tools": DISABLED_TOOLS}, + temperature=0.6, + extra_body={"tool_set": TOOL_SET}, ) - summary = summary_response.choices[0].message if summary_response.choices else None - if summary: + if summary_response.choices: + summary = summary_response.choices[0].message conversation_messages.append(summary.model_dump(exclude_none=True)) final_answer = summary.content or None except Exception as summary_error: @@ -225,6 +178,52 @@ async def sampling_loop( } +async def _run_tool_call( + tool_call: Any, + computer_tool: ComputerTool, + system_tools: SystemTools, +) -> dict[str, Any]: + name = tool_call.function.name + + try: + args = json.loads(tool_call.function.arguments) + except json.JSONDecodeError: + print(f"Failed to parse tool_call arguments: {tool_call.function.arguments}") + return { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "Error: failed to parse arguments", + } + + print(f"Executing tool: {name}", json.dumps(args)[:500]) + + if name == "computer_batch": + outcome = await computer_tool.run_batch(args.get("actions") or []) + return { + "role": "tool", + "tool_call_id": tool_call.id, + "content": [ + {"type": "text", "text": outcome.describe()}, + _image_part(await computer_tool.screenshot()), + ], + } + + try: + content = system_tools.execute(name, args) + except Exception as error: + print(f"{name} failed: {error}") + content = f"{name} failed: {error}" + + return {"role": "tool", "tool_call_id": tool_call.id, "content": content} + + +def _image_part(base64_image: str) -> dict[str, Any]: + return { + "type": "image_url", + "image_url": {"url": f"data:image/webp;base64,{base64_image}"}, + } + + def _format_task_with_context(task: str, user_timezone: str, user_location: str) -> str: """Append location, timezone, and current date/time to the task message.""" for timezone_name in [user_timezone, "America/Los_Angeles", "UTC"]: @@ -260,48 +259,28 @@ def _format_stop_and_summarize(task: str) -> str: ) -def _trimmed_for_request( - messages: list[dict[str, Any]], -) -> tuple[list[dict[str, Any]], int]: - """Return a deep-copied messages list with old screenshots stripped to fit MAX_REQUEST_BYTES. +def _trimmed_for_request(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Drop the screenshots n2 will not look at, keeping the text of every turn. - The most recent KEEP_RECENT_SCREENSHOTS screenshots are protected. The full - `messages` list is preserved unchanged for the caller's return value. + The model reads images from the last KEEP_IMAGE_MESSAGES image-bearing + messages and ignores the rest, so those are stripped every request. If the + payload is still over the cap after that, the previous screenshot goes too — + the latest one is never dropped, since a request without one badly degrades + grounding. """ trimmed = copy.deepcopy(messages) - size = _estimate_size(trimmed) - if size <= MAX_REQUEST_BYTES: - return trimmed, 0 - image_indices = [i for i, m in enumerate(trimmed) if _message_has_image(m)] - if not image_indices: - return trimmed, 0 - protected = set(image_indices[-max(1, KEEP_RECENT_SCREENSHOTS):]) - removed = 0 + for idx in image_indices[:-KEEP_IMAGE_MESSAGES]: + _strip_images(trimmed[idx]) - for idx in image_indices: - if size <= MAX_REQUEST_BYTES: + for idx in image_indices[-KEEP_IMAGE_MESSAGES:-1]: + if _estimate_size(trimmed) <= MAX_REQUEST_BYTES: break - if idx in protected: - continue - if _strip_one_image(trimmed[idx]): - removed += 1 - size = _estimate_size(trimmed) - - # If still over, strip from the protected window too — but always keep the latest. - if size > MAX_REQUEST_BYTES: - last_idx = image_indices[-1] - for idx in image_indices: - if size <= MAX_REQUEST_BYTES: - break - if idx == last_idx: - continue - if _strip_one_image(trimmed[idx]): - removed += 1 - size = _estimate_size(trimmed) + print("Payload still over the 10 MB cap — dropping the previous screenshot too") + _strip_images(trimmed[idx]) - return trimmed, removed + return trimmed def _estimate_size(messages: list[dict[str, Any]]) -> int: @@ -315,50 +294,18 @@ def _message_has_image(msg: dict[str, Any]) -> bool: return any(isinstance(p, dict) and p.get("type") == "image_url" for p in content) -def _strip_one_image(msg: dict[str, Any]) -> bool: +def _strip_images(msg: dict[str, Any]) -> None: content = msg.get("content") if not isinstance(content, list): - return False - - removed = False - new_content: list[dict[str, Any]] = [] - for part in content: - if not removed and isinstance(part, dict) and part.get("type") == "image_url": - removed = True - continue - new_content.append(part) + return - if not removed: - return False + new_content = [ + part for part in content + if not (isinstance(part, dict) and part.get("type") == "image_url") + ] has_text = any(isinstance(p, dict) and p.get("type") == "text" for p in new_content) if not has_text: - new_content.append({"type": "text", "text": "Screenshot omitted to stay under request size limit."}) + new_content.append({"type": "text", "text": "Screenshot omitted — superseded by a newer one."}) msg["content"] = new_content - return True - - -def _scale_coordinates(action: N15Action, viewport_width: int, viewport_height: int) -> N15Action: - scaled = dict(action) - - if "coordinates" in scaled and scaled["coordinates"]: - scaled["coordinates"] = _denormalize(scaled["coordinates"], viewport_width, viewport_height) - - if "start_coordinates" in scaled and scaled["start_coordinates"]: - scaled["start_coordinates"] = _denormalize(scaled["start_coordinates"], viewport_width, viewport_height) - - return scaled - - -def _denormalize(coords: list[int] | tuple[int, int], width: int, height: int) -> list[int]: - """Map [0, 1000] coordinates to viewport pixels and clamp to [0, dim-1]. - - Clamping prevents a boundary value like 1000 from landing one pixel outside - the viewport on a 1280x800 display. - """ - raw_x = round((coords[0] / NAVIGATOR_COORDINATE_SCALE) * width) - raw_y = round((coords[1] / NAVIGATOR_COORDINATE_SCALE) * height) - x = max(0, min(width - 1, raw_x)) - y = max(0, min(height - 1, raw_y)) - return [x, y] diff --git a/pkg/templates/python/yutori/main.py b/pkg/templates/python/yutori/main.py index 94d13e10..cacf3bd8 100644 --- a/pkg/templates/python/yutori/main.py +++ b/pkg/templates/python/yutori/main.py @@ -8,7 +8,7 @@ class _QueryInputOptional(TypedDict, total=False): record_replay: Optional[bool] - kiosk: Optional[bool] + reasoning_effort: Optional[str] user_timezone: Optional[str] user_location: Optional[str] @@ -35,14 +35,14 @@ async def cua_task( payload: QueryInput, ) -> QueryOutput: """ - Process a user query using Yutori n1.5 Computer Use with Kernel's browser automation. + Process a user query using Yutori n2 Computer Use with Kernel's browser automation. Args: ctx: Kernel context containing invocation information payload: An object containing: - query: The task/query string to process - record_replay: Optional boolean to enable video replay recording - - kiosk: Optional boolean to launch in kiosk mode + - reasoning_effort: Optional "none" | "low" | "medium" | "xhigh" - user_timezone: Optional IANA tz (e.g. "America/New_York") - user_location: Optional free-text location for model context @@ -54,27 +54,26 @@ async def cua_task( if not payload or not payload.get("query"): raise ValueError("Query is required") - record_replay = payload.get("record_replay", False) - kiosk_mode = payload.get("kiosk", False) - async with KernelBrowserSession( invocation_id=ctx.invocation_id, stealth=True, - record_replay=record_replay, - kiosk_mode=kiosk_mode, + record_replay=payload.get("record_replay", False), ) as session: print("Kernel browser live view url:", session.live_view_url) loop_kwargs: dict = { - "model": "n1.5-latest", + "model": "n2", "task": payload["query"], "api_key": str(api_key), "kernel": session.kernel, "session_id": str(session.session_id), - "viewport_width": session.viewport_width, - "viewport_height": session.viewport_height, - "kiosk_mode": kiosk_mode, + # n2 sees the whole screen, browser chrome included — the Kernel + # viewport is the window size, so it is the screen size here. + "screen_width": session.viewport_width, + "screen_height": session.viewport_height, } + if payload.get("reasoning_effort"): + loop_kwargs["reasoning_effort"] = payload["reasoning_effort"] if payload.get("user_timezone"): loop_kwargs["user_timezone"] = payload["user_timezone"] if payload.get("user_location"): diff --git a/pkg/templates/python/yutori/pyproject.toml b/pkg/templates/python/yutori/pyproject.toml index 87c17c70..7b82ec2e 100644 --- a/pkg/templates/python/yutori/pyproject.toml +++ b/pkg/templates/python/yutori/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "python-yutori-cua" version = "0.1.0" -description = "Kernel reference app for Yutori n1.5 Computer Use" +description = "Kernel reference app for Yutori n2 Computer Use" requires-python = ">=3.9" dependencies = [ "openai>=1.58.0", diff --git a/pkg/templates/python/yutori/session.py b/pkg/templates/python/yutori/session.py index 9bf020f2..93de7c43 100644 --- a/pkg/templates/python/yutori/session.py +++ b/pkg/templates/python/yutori/session.py @@ -39,9 +39,6 @@ class KernelBrowserSession: record_replay: bool = False replay_grace_period: float = 5.0 # Seconds to wait before stopping replay - # Kiosk mode (hides address bar and tabs in live view) - kiosk_mode: bool = False - # Invocation ID to link browser session to the action invocation invocation_id: Optional[str] = None @@ -64,7 +61,6 @@ async def __aenter__(self) -> "KernelBrowserSession": "width": self.viewport_width, "height": self.viewport_height, }, - kiosk_mode=self.kiosk_mode, ) self.session_id = browser.session_id diff --git a/pkg/templates/python/yutori/tools/__init__.py b/pkg/templates/python/yutori/tools/__init__.py index 5a1a4285..8a8394f3 100644 --- a/pkg/templates/python/yutori/tools/__init__.py +++ b/pkg/templates/python/yutori/tools/__init__.py @@ -1,11 +1,13 @@ -"""Yutori n1.5 Computer Tools.""" +"""Yutori n2 Computer Use Tools.""" -from .base import ToolError, ToolResult -from .computer import ComputerTool, N15Action +from .base import ToolError +from .computer import BatchOutcome, ComputerTool, N2Action +from .system import SystemTools __all__ = [ "ToolError", - "ToolResult", + "BatchOutcome", "ComputerTool", - "N15Action", + "N2Action", + "SystemTools", ] diff --git a/pkg/templates/python/yutori/tools/base.py b/pkg/templates/python/yutori/tools/base.py index d12a38e6..1a730994 100644 --- a/pkg/templates/python/yutori/tools/base.py +++ b/pkg/templates/python/yutori/tools/base.py @@ -1,6 +1,4 @@ -"""Base tool types for Yutori n1.""" - -from typing import TypedDict +"""Base tool types for Yutori n2.""" class ToolError(Exception): @@ -9,9 +7,3 @@ class ToolError(Exception): def __init__(self, message: str): self.message = message super().__init__(message) - - -class ToolResult(TypedDict, total=False): - base64_image: str - output: str - error: str diff --git a/pkg/templates/python/yutori/tools/computer.py b/pkg/templates/python/yutori/tools/computer.py index d4debded..e6085cd5 100644 --- a/pkg/templates/python/yutori/tools/computer.py +++ b/pkg/templates/python/yutori/tools/computer.py @@ -1,80 +1,96 @@ """ -Yutori n1.5 Computer Tool +Yutori n2 Computer Tool -Maps n1.5-latest action format to Kernel's Computer Controls API. -Screenshots are converted to WebP for better compression across multi-step trajectories. +Maps n2's `computer_batch` action list onto Kernel's Computer Controls API. +n2 plans every action in a batch against the screenshot taken *before* the batch +ran, so the batch executes sequentially, stops at the first error, and answers +with a single screenshot taken after the last action that ran. -@see https://docs.yutori.com/reference/n1-5 +@see https://docs.yutori.com/reference/n2 """ from __future__ import annotations import asyncio import base64 -import json +from dataclasses import dataclass from io import BytesIO -from typing import Any, Literal, TypedDict +from typing import Any, Literal, Optional, TypedDict from kernel import Kernel from PIL import Image -from .base import ToolError, ToolResult +from .base import ToolError TYPING_DELAY_MS = 12 -SCREENSHOT_DELAY_S = 0.15 -ACTION_DELAY_S = 0.3 +# Let the UI settle between actions in a batch, and again before the screenshot +# that the model will plan its next batch against. +INTER_ACTION_DELAY_S = 0.08 +SETTLE_DELAY_S = 0.4 -# n1.5 scroll `amount` is in "wheel units" where 1 unit ≈ 10% of the viewport -# height (~80px at 800px tall). Kernel's delta_y is a wheel-event repeat count -# where each tick is much smaller in practice, so we multiply. -SCROLL_NOTCHES_PER_AMOUNT = 4 +NAVIGATOR_COORDINATE_SCALE = 1000 + +# n2's scroll `amount` is a wheel notch count (1-50), which is exactly what +# Kernel's delta_x/delta_y take ("xdotool wheel units"), so it passes through. +DEFAULT_SCROLL_AMOUNT = 3 # WebP quality for screenshots. Kernel returns PNGs, which are crisp and # tolerate aggressive WebP compression with no visible degradation — matches # Yutori SDK's DEFAULT_WEBP_QUALITY_FOR_PNG=30 (yutori-sdk-python/yutori/ -# navigator/images.py). Lower values cut payload size substantially on long -# multi-step trajectories. +# navigator/images.py). Requests are capped at 10 MB and a trajectory carries +# several full-screen captures, so the compression matters. WEBP_QUALITY = 30 -N15ActionType = Literal[ +N2ActionName = Literal[ "left_click", "double_click", "triple_click", "middle_click", "right_click", - "mouse_move", - "mouse_down", - "mouse_up", "scroll", "type", "key_press", - "hold_key", "drag", + "mouse_move", + "mouse_down", + "mouse_up", + "hold_key", "wait", - "refresh", - "go_back", - "go_forward", - "goto_url", + "screenshot", ] -class N15Action(TypedDict, total=False): - action_type: N15ActionType - coordinates: tuple[int, int] | list[int] - start_coordinates: tuple[int, int] | list[int] - direction: Literal["up", "down", "left", "right"] - amount: int - text: str - key: str - modifier: str - duration: int - url: str +class N2Action(TypedDict, total=False): + """One `{name, arguments}` member of a `computer_batch` call.""" + + name: N2ActionName + arguments: dict[str, Any] + + +@dataclass +class BatchOutcome: + executed: int + total: int + # Set when an action raised; the remaining actions were skipped. + failed_index: Optional[int] = None + failed_name: Optional[str] = None + failed_message: Optional[str] = None + + def describe(self) -> str: + if self.failed_index is None: + return f"Executed {self.executed} of {self.total} actions." + return ( + f"Executed {self.executed} of {self.total} actions. " + f"Action {self.failed_index + 1} ({self.failed_name}) failed: {self.failed_message}. " + f"The remaining actions were skipped." + ) -# n1.5 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). +# n2 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). # Kernel's press_key expects XKeysym names (e.g. `Return`, `Ctrl`, `Page_Up`). -# Keys not in the map pass through unchanged (printable characters like `a`, -# `1`, `,` are already XKeysym). +# This map covers every key Yutori documents at +# https://docs.yutori.com/reference/n2#key-space — keys not in the map pass +# through unchanged (printable characters like `a` and `1` are already XKeysym). # # Sister implementation (Playwright target instead of XKeysym): # https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/keys.py @@ -115,6 +131,19 @@ class N15Action(TypedDict, total=False): "pagedown": "Page_Down", # Function keys **{f"f{i}": f"F{i}" for i in range(1, 13)}, + # Punctuation — n2 sends word forms, most of which are already XKeysym names + "minus": "minus", + "plus": "plus", + "equal": "equal", + "comma": "comma", + "period": "period", + "slash": "slash", + "backslash": "backslash", + "semicolon": "semicolon", + "quote": "apostrophe", + "backquote": "grave", + "bracketleft": "bracketleft", + "bracketright": "bracketright", # Locks / special "capslock": "Caps_Lock", "numlock": "Num_Lock", @@ -126,19 +155,11 @@ class N15Action(TypedDict, total=False): def _map_token(token: str) -> str: - lower = token.strip().lower() - return KEY_MAP.get(lower, token.strip()) - - -def _normalize_url(url: str) -> str: - trimmed = url.strip() - if "://" in trimmed: - return trimmed - return f"https://{trimmed}" + return KEY_MAP.get(token.strip().lower(), token.strip()) def _parse_key_expression(expr: str) -> list[str]: - """Parse an n1.5 key expression into one Kernel combo per sequential press. + """Parse an n2 key expression into one Kernel combo per sequential press. Spaces separate sequential presses; '+' separates simultaneous tokens within a press. Examples: @@ -154,285 +175,198 @@ def _parse_key_expression(expr: str) -> list[str]: ] +def _duration_ms(duration: Any, fallback_ms: int) -> int: + """n2 emits `duration` in seconds; Kernel takes milliseconds.""" + if isinstance(duration, (int, float)) and duration > 0: + return int(duration * 1000) + return fallback_ms + + class ComputerTool: - def __init__(self, kernel: Kernel, session_id: str, width: int = 1280, height: int = 800, kiosk_mode: bool = False): + def __init__(self, kernel: Kernel, session_id: str, screen_width: int = 1280, screen_height: int = 800): self.kernel = kernel self.session_id = session_id - self.width = width - self.height = height - self.kiosk_mode = kiosk_mode - - async def execute(self, action: N15Action) -> ToolResult: - action_type = action.get("action_type") - - handlers = { - "left_click": lambda a: self._handle_click(a, "left", 1), - "double_click": lambda a: self._handle_click(a, "left", 2), - "triple_click": lambda a: self._handle_click(a, "left", 3), - "middle_click": lambda a: self._handle_click(a, "middle", 1), - "right_click": lambda a: self._handle_click(a, "right", 1), - "mouse_move": self._handle_mouse_move, - "mouse_down": lambda a: self._handle_mouse_button(a, "down"), - "mouse_up": lambda a: self._handle_mouse_button(a, "up"), - "scroll": self._handle_scroll, - "type": self._handle_type, - "key_press": self._handle_key_press, - "hold_key": self._handle_hold_key, - "drag": self._handle_drag, - "wait": self._handle_wait, - "refresh": self._handle_refresh, - "go_back": self._handle_go_back, - "go_forward": self._handle_go_forward, - "goto_url": self._handle_goto_url, - } - - handler = handlers.get(action_type) - if not handler: - raise ToolError(f"Unknown action type: {action_type}") - - return await handler(action) - - async def _handle_click(self, action: N15Action, button: str, num_clicks: int) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) - modifier = action.get("modifier") + self.screen_width = screen_width + self.screen_height = screen_height + + async def run_batch(self, actions: list[N2Action]) -> BatchOutcome: + """Run a `computer_batch` action list in order, stopping at the first failure. + + The caller reports the outcome and a single post-batch screenshot back to n2. + """ + for index, action in enumerate(actions): + try: + await self._run_action(action) + except Exception as error: + return BatchOutcome( + executed=index, + total=len(actions), + failed_index=index, + failed_name=str(action.get("name")), + failed_message=str(error), + ) + if index < len(actions) - 1: + await asyncio.sleep(INTER_ACTION_DELAY_S) + + return BatchOutcome(executed=len(actions), total=len(actions)) + + async def _run_action(self, action: N2Action) -> None: + name = action.get("name") + args = action.get("arguments") or {} + + if name in ("left_click", "double_click", "triple_click"): + num_clicks = {"left_click": 1, "double_click": 2, "triple_click": 3}[name] + return self._click(args, "left", num_clicks) + if name == "middle_click": + return self._click(args, "middle", 1) + if name == "right_click": + return self._click(args, "right", 1) + if name == "scroll": + return self._scroll(args) + if name == "type": + return self._type(args) + if name == "key_press": + return self._key_press(args) + if name == "drag": + return self._drag(args) + if name == "mouse_move": + return self._mouse_move(args) + if name in ("mouse_down", "mouse_up"): + return self._mouse_button(args, name.split("_")[1]) + if name == "hold_key": + return self._hold_key(args) + if name == "wait": + return await asyncio.sleep(_duration_ms(args.get("duration"), 2000) / 1000) + if name == "screenshot": + # A batch already answers with a screenshot taken after its last + # action, so an explicit `screenshot` member has nothing to do. + return None + + raise ToolError(f"Unknown action: {name}") + + def _click(self, args: dict[str, Any], button: str, num_clicks: int) -> None: + x, y = self._require_coordinates(args.get("coordinates")) kwargs: dict[str, Any] = { - "x": coords["x"], - "y": coords["y"], + "x": x, + "y": y, "button": button, "click_type": "click", "num_clicks": num_clicks, } - if modifier: - kwargs["hold_keys"] = [_map_token(modifier)] + if args.get("modifier"): + kwargs["hold_keys"] = [_map_token(args["modifier"])] self.kernel.browsers.computer.click_mouse(self.session_id, **kwargs) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_mouse_move(self, action: N15Action) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) + def _mouse_move(self, args: dict[str, Any]) -> None: + x, y = self._require_coordinates(args.get("coordinates")) + self.kernel.browsers.computer.move_mouse(self.session_id, x=x, y=y) - self.kernel.browsers.computer.move_mouse( - self.session_id, - x=coords["x"], - y=coords["y"], - ) - - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_mouse_button(self, action: N15Action, click_type: str) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) + def _mouse_button(self, args: dict[str, Any], click_type: str) -> None: + # Coordinates are optional here — without them the button is pressed or + # released wherever the cursor already is, which is what a manual + # mouse_move -> mouse_down -> mouse_move -> mouse_up drag relies on. + if args.get("coordinates"): + x, y = self._require_coordinates(args["coordinates"]) + else: + position = self.kernel.browsers.computer.get_mouse_position(self.session_id) + x, y = position.x, position.y self.kernel.browsers.computer.click_mouse( self.session_id, - x=coords["x"], - y=coords["y"], + x=x, + y=y, button="left", click_type=click_type, ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_scroll(self, action: N15Action) -> ToolResult: - coords = self._get_coordinates(action.get("coordinates")) - direction = action.get("direction") - amount = max(action.get("amount", 3), 1) + def _scroll(self, args: dict[str, Any]) -> None: + x, y = self._require_coordinates(args.get("coordinates")) + direction = args.get("direction") - if direction not in ("up", "down", "left", "right"): + # n2 only scrolls vertically. + if direction not in ("up", "down"): raise ToolError(f"Invalid scroll direction: {direction}") - # Yutori 1 unit ≈ 10% of viewport height; scale into Kernel wheel-event ticks. - ticks = amount * SCROLL_NOTCHES_PER_AMOUNT - - delta_x = 0 - delta_y = 0 - - if direction == "up": - delta_y = -ticks - elif direction == "down": - delta_y = ticks - elif direction == "left": - delta_x = -ticks - elif direction == "right": - delta_x = ticks - - modifier = action.get("modifier") - scroll_kwargs: dict[str, Any] = { - "x": coords["x"], - "y": coords["y"], - "delta_x": delta_x, - "delta_y": delta_y, + notches = max(1, round(args.get("amount") or DEFAULT_SCROLL_AMOUNT)) + kwargs: dict[str, Any] = { + "x": x, + "y": y, + "delta_x": 0, + "delta_y": -notches if direction == "up" else notches, } - if modifier: - scroll_kwargs["hold_keys"] = [_map_token(modifier)] - - self.kernel.browsers.computer.scroll(self.session_id, **scroll_kwargs) + if args.get("modifier"): + kwargs["hold_keys"] = [_map_token(args["modifier"])] - await asyncio.sleep(SCREENSHOT_DELAY_S) - screenshot_result = await self.screenshot() - screenshot_result["output"] = f"Scrolled {amount} unit(s) {direction}." - return screenshot_result + self.kernel.browsers.computer.scroll(self.session_id, **kwargs) - async def _handle_type(self, action: N15Action) -> ToolResult: - text = action.get("text") + def _type(self, args: dict[str, Any]) -> None: + text = args.get("text") if not text: - raise ToolError("text is required for type action") + raise ToolError("text is required for type") - self.kernel.browsers.computer.type_text( - self.session_id, - text=text, - delay=TYPING_DELAY_MS, - ) + self.kernel.browsers.computer.type_text(self.session_id, text=text, delay=TYPING_DELAY_MS) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_key_press(self, action: N15Action) -> ToolResult: - key = action.get("key") + def _key_press(self, args: dict[str, Any]) -> None: + key = args.get("key") if not key: - raise ToolError("key is required for key_press action") + raise ToolError("key is required for key_press") - # n1.5 supports sequential presses ("down down down enter") — issue each + # n2 supports sequential presses ("down down down enter") — issue each # combo as its own press_key so they're seen as separate keystrokes. - combos = _parse_key_expression(key) - for combo in combos: + for combo in _parse_key_expression(key): self.kernel.browsers.computer.press_key(self.session_id, keys=[combo]) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_hold_key(self, action: N15Action) -> ToolResult: - key = action.get("key") + def _hold_key(self, args: dict[str, Any]) -> None: + key = args.get("key") if not key: - raise ToolError("key is required for hold_key action") - - # Yutori emits `duration` in seconds; Kernel SDK's press_key takes ms. - duration_s = action.get("duration") - duration_ms = int(duration_s * 1000) if duration_s and duration_s > 0 else 1000 + raise ToolError("key is required for hold_key") - combos = _parse_key_expression(key) - for combo in combos: + for combo in _parse_key_expression(key): self.kernel.browsers.computer.press_key( self.session_id, keys=[combo], - duration=duration_ms, + duration=_duration_ms(args.get("duration"), 1000), ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_drag(self, action: N15Action) -> ToolResult: - start_coords = self._get_coordinates(action.get("start_coordinates")) - end_coords = self._get_coordinates(action.get("coordinates")) + def _drag(self, args: dict[str, Any]) -> None: + start_x, start_y = self._require_coordinates(args.get("start_coordinates")) + end_x, end_y = self._require_coordinates(args.get("coordinates")) self.kernel.browsers.computer.drag_mouse( self.session_id, - path=[[start_coords["x"], start_coords["y"]], [end_coords["x"], end_coords["y"]]], + path=[[start_x, start_y], [end_x, end_y]], button="left", ) - await asyncio.sleep(SCREENSHOT_DELAY_S) - return await self.screenshot() - - async def _handle_wait(self, action: N15Action) -> ToolResult: - # Yutori emits `duration` in seconds (matches reference impl). - duration = action.get("duration") - seconds = duration if duration and duration > 0 else 2 - await asyncio.sleep(seconds) - return await self.screenshot() - - async def _handle_refresh(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["F5"], - ) - await asyncio.sleep(2) - return await self.screenshot() - - async def _handle_go_back(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Alt+Left"], - ) - await asyncio.sleep(1.5) - return await self.screenshot() + async def screenshot(self) -> str: + """Capture the whole screen — browser chrome included — as base64 WebP.""" + await asyncio.sleep(SETTLE_DELAY_S) - async def _handle_go_forward(self, action: N15Action) -> ToolResult: - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Alt+Right"], - ) - await asyncio.sleep(1.5) - return await self.screenshot() + response = self.kernel.browsers.computer.capture_screenshot(self.session_id) + image = Image.open(BytesIO(response.read())) + webp_buffer = BytesIO() + image.save(webp_buffer, "WEBP", quality=WEBP_QUALITY) - async def _handle_goto_url(self, action: N15Action) -> ToolResult: - url = action.get("url") - if not url: - raise ToolError("url is required for goto_url action") - target_url = _normalize_url(url) + return base64.b64encode(webp_buffer.getvalue()).decode("utf-8") - if self.kiosk_mode: - response = self.kernel.browsers.playwright.execute( - self.session_id, - code=f"await page.goto({json.dumps(target_url)});", - timeout_sec=60, - ) - if not response.success: - raise ToolError(response.error or "Playwright goto failed") - await asyncio.sleep(ACTION_DELAY_S) - return await self.screenshot() + def _require_coordinates(self, coords: Any) -> tuple[int, int]: + """Map [0, 1000] coordinates to screen pixels, clamped to [0, dim-1]. - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Ctrl+l"], - ) - await asyncio.sleep(ACTION_DELAY_S) + Clamping prevents a boundary value like 1000 from landing one pixel + outside the screen on a 1280x800 display. + """ + if not isinstance(coords, (list, tuple)) or len(coords) != 2: + raise ToolError(f"coordinates are required, got {coords!r}") - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Ctrl+a"], - ) - await asyncio.sleep(0.1) + nx, ny = coords + if not isinstance(nx, (int, float)) or not isinstance(ny, (int, float)): + raise ToolError(f"Invalid coordinates: {coords!r}") - self.kernel.browsers.computer.type_text( - self.session_id, - text=target_url, - delay=TYPING_DELAY_MS, - ) - await asyncio.sleep(ACTION_DELAY_S) + x = round(nx / NAVIGATOR_COORDINATE_SCALE * self.screen_width) + y = round(ny / NAVIGATOR_COORDINATE_SCALE * self.screen_height) - self.kernel.browsers.computer.press_key( - self.session_id, - keys=["Return"], + return ( + max(0, min(self.screen_width - 1, x)), + max(0, min(self.screen_height - 1, y)), ) - await asyncio.sleep(2) - return await self.screenshot() - - async def screenshot(self) -> ToolResult: - try: - response = self.kernel.browsers.computer.capture_screenshot( - self.session_id - ) - png_bytes = response.read() - img = Image.open(BytesIO(png_bytes)) - webp_buf = BytesIO() - img.save(webp_buf, "WEBP", quality=WEBP_QUALITY) - base64_image = base64.b64encode(webp_buf.getvalue()).decode("utf-8") - return {"base64_image": base64_image} - except Exception as e: - raise ToolError(f"Failed to take screenshot: {e}") - - def _get_coordinates( - self, coords: tuple[int, int] | list[int] | None - ) -> dict[str, int]: - if coords is None or len(coords) != 2: - return {"x": self.width // 2, "y": self.height // 2} - - x, y = coords - if not isinstance(x, (int, float)) or not isinstance(y, (int, float)) or x < 0 or y < 0: - raise ToolError(f"Invalid coordinates: {coords}") - - return {"x": int(x), "y": int(y)} diff --git a/pkg/templates/python/yutori/tools/system.py b/pkg/templates/python/yutori/tools/system.py new file mode 100644 index 00000000..a29e2d81 --- /dev/null +++ b/pkg/templates/python/yutori/tools/system.py @@ -0,0 +1,184 @@ +""" +Yutori n2 Shell and File Tools + +n2's tool set always ships `bash`, `read`, `write`, and `edit` alongside +`computer_batch` — `disable_tools` is rejected by the API, so a loop has to +answer these too. They run against the Kernel browser VM via the process and +filesystem APIs. + +@see https://docs.yutori.com/reference/n2 +""" + +from __future__ import annotations + +import base64 +import shlex +import time +from typing import Any, Optional + +from kernel import Kernel + +from .base import ToolError + +DEFAULT_TIMEOUT_SEC = 120 +MAX_TIMEOUT_SEC = 600 +DEFAULT_READ_LIMIT = 2000 +MAX_WRITE_CHARS = 256_000 + +# bash: "The working directory persists across calls; environment variables and +# shell functions do not." Each exec is its own process, so the command reports +# its final directory on a sentinel line that we strip before returning stdout. +CWD_SENTINEL = "__n2_cwd__" + + +def _decode(value: Optional[str]) -> str: + return base64.b64decode(value).decode("utf-8", errors="replace") if value else "" + + +def _format_command_result(stdout: str, stderr: str, exit_code: Optional[int]) -> str: + parts = [] + if stdout.strip(): + parts.append(stdout.rstrip()) + if stderr.strip(): + parts.append(f"stderr:\n{stderr.rstrip()}") + if exit_code: + parts.append(f"Exited with code {exit_code}.") + + return "\n".join(parts) if parts else "Command produced no output." + + +class SystemTools: + def __init__(self, kernel: Kernel, session_id: str): + self.kernel = kernel + self.session_id = session_id + self.cwd: Optional[str] = None + self.seen_paths: set[str] = set() + + def execute(self, name: str, args: dict[str, Any]) -> str: + handlers = { + "bash": self._bash, + "read": self._read, + "write": self._write, + "edit": self._edit, + } + handler = handlers.get(name) + if not handler: + raise ToolError(f"Unknown tool: {name}") + + return handler(args) + + def _bash(self, args: dict[str, Any]) -> str: + command = args.get("command") + if not command: + raise ToolError("command is required for bash") + + if args.get("run_in_background"): + return self._bash_background(command) + + script = "\n".join([ + command, + "n2_status=$?", + f"printf '\\n{CWD_SENTINEL}%s' \"$(pwd)\"", + "exit $n2_status", + ]) + + result = self.kernel.browsers.process.exec( + self.session_id, + command="bash", + args=["-lc", script], + cwd=self.cwd, + timeout_sec=min(args.get("timeout") or DEFAULT_TIMEOUT_SEC, MAX_TIMEOUT_SEC), + ) + + stdout = self._take_cwd(_decode(result.stdout_b64)) + + return _format_command_result(stdout, _decode(result.stderr_b64), result.exit_code) + + def _read(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + if not file_path: + raise ToolError("file_path is required for read") + + response = self.kernel.browsers.fs.read_file(self.session_id, path=file_path) + lines = response.read().decode("utf-8", errors="replace").split("\n") + + offset = max(0, args.get("offset") or 0) + limit = max(1, args.get("limit") or DEFAULT_READ_LIMIT) + page = lines[offset : offset + limit] + + self.seen_paths.add(file_path) + + if not page: + return f"{file_path} has {len(lines)} line(s); offset {offset} is past the end." + + # cat -n format, so line numbers survive into the model's next edit. + return "\n".join(f"{offset + i + 1:>6}\t{line}" for i, line in enumerate(page)) + + def _write(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + content = args.get("content") + if not file_path or content is None: + raise ToolError("file_path and content are required for write") + if len(content) > MAX_WRITE_CHARS: + raise ToolError(f"content exceeds the {MAX_WRITE_CHARS} character cap") + + self.kernel.browsers.fs.write_file(self.session_id, content.encode("utf-8"), path=file_path) + self.seen_paths.add(file_path) + + return f"Wrote {len(content)} character(s) to {file_path}." + + def _edit(self, args: dict[str, Any]) -> str: + file_path = args.get("file_path") + old_string = args.get("old_string") + new_string = args.get("new_string") + if not file_path or old_string is None or new_string is None: + raise ToolError("file_path, old_string, and new_string are required for edit") + # n2 is expected to know the current bytes before changing them. + if file_path not in self.seen_paths: + raise ToolError(f"{file_path} has not been read in this session — read it before editing.") + + response = self.kernel.browsers.fs.read_file(self.session_id, path=file_path) + original = response.read().decode("utf-8", errors="replace") + + occurrences = original.count(old_string) + if occurrences == 0: + raise ToolError(f"old_string not found in {file_path}") + + replace_all = bool(args.get("replace_all")) + if occurrences > 1 and not replace_all: + raise ToolError( + f"old_string matches {occurrences} times in {file_path} — " + f"pass replace_all or include more context." + ) + + updated = original.replace(old_string, new_string, -1 if replace_all else 1) + self.kernel.browsers.fs.write_file(self.session_id, updated.encode("utf-8"), path=file_path) + + return f"Replaced {occurrences if replace_all else 1} occurrence(s) in {file_path}." + + def _bash_background(self, command: str) -> str: + log_path = f"/tmp/n2-bg-{int(time.time() * 1000)}.log" + script = f"nohup bash -c {shlex.quote(command)} > {log_path} 2>&1 &\necho $!" + + result = self.kernel.browsers.process.exec( + self.session_id, + command="bash", + args=["-lc", script], + cwd=self.cwd, + ) + pid = _decode(result.stdout_b64).strip() + + return "\n".join([ + f"Started in the background with pid {pid}.", + f"Output is being written to {log_path} — use the read tool to check on it.", + f"Cancel it with: kill {pid}", + ]) + + def _take_cwd(self, stdout: str) -> str: + """Strip the trailing sentinel line and remember the directory it reported.""" + marker = stdout.rfind(f"\n{CWD_SENTINEL}") + if marker == -1: + return stdout + + self.cwd = stdout[marker + len(CWD_SENTINEL) + 1 :].strip() or self.cwd + return stdout[:marker] diff --git a/pkg/templates/typescript/yutori/README.md b/pkg/templates/typescript/yutori/README.md index 04be089f..477f1127 100644 --- a/pkg/templates/typescript/yutori/README.md +++ b/pkg/templates/typescript/yutori/README.md @@ -1,10 +1,8 @@ -# Kernel TypeScript Sample App - Yutori n1.5 Computer Use +# Kernel TypeScript Sample App - Yutori n2 Computer Use -This Kernel app implements a prompt loop using Yutori's Navigator n1.5 with Kernel's Computer Controls API. +This Kernel app implements a prompt loop using Yutori's Navigator n2 with Kernel's Computer Controls API. -[Navigator n1.5](https://yutori.com/blog/introducing-n1-5) is Yutori's pixels-to-actions LLM that predicts browser actions from screenshots. - -This template runs n1.5 in **computer-use-only mode**. n1.5 also supports a hybrid vision + DOM/JavaScript path (page-state extraction, custom JS, structured JSON output) for multi-field forms and bulk data extraction, but those tools are intentionally disabled here — see [Disabled tools](#disabled-tools). +[Navigator n2](https://docs.yutori.com/reference/n2) is Yutori's computer-use model. It reads a screenshot of the whole screen and answers with a batch of mouse and keyboard actions, and it can run shell commands and edit files on the machine it is driving. ## Setup @@ -28,7 +26,7 @@ kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://w Optional payload fields: - `record_replay` (bool) — capture a video of the session (paid plans only). -- `kiosk` (bool) — launch the browser without address bar / tabs ([see below](#kiosk-mode)). +- `reasoning_effort` (`"none"`, `"low"`, `"medium"`, `"xhigh"`) — n2 reasons at `medium` by default. `xhigh` gives the longest traces and does best on hard multi-step tasks; `none` turns reasoning off. - `user_timezone` (IANA, e.g. `"America/New_York"`) and `user_location` (free text, e.g. `"New York, NY, US"`) — appended to the task message so the model has accurate temporal/locational grounding. More involved example (Kanban drag-and-drop): @@ -49,66 +47,62 @@ kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://e When enabled, the response will include a `replay_url` field with a link to view the recorded session. -## Kiosk mode - -Prefer **non-kiosk mode** by default and when the agent is expected to switch domains via URL. Use **kiosk (`"kiosk": true`)** when: (1) you're recording sessions and want a cleaner UI in the replay, or (2) you're automating on a single website and the combination of the complex site layout and browser chrome (address bar, tabs) may confuse the agent. - -Note: In kiosk mode the agent may still try to use the address bar to enter URLs; it's not available, so it will eventually use `goto_url`, but those attempts may result in slowdown of the overall session. +## Screen Configuration -Default (non-kiosk): - -```bash -kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com, then navigate to ign.com and describe the page"}' -``` +n2 is a desktop model: it expects a screenshot of the **whole screen**, browser chrome included, and it navigates by clicking the address bar rather than through a dedicated navigation action. A Kernel `viewport` sets the browser window size, and screenshots from Computer Controls capture that window with its tabs and toolbar — which is exactly what n2 wants, so this template does not use kiosk mode. -With kiosk (single-site or recording): +This template runs at **1280x800**. Yutori also lists 1920x1080 and 1280x720 as resolutions in regular use; grounding may degrade at extreme aspect ratios. -```bash -kernel invoke ts-yutori-cua cua-task --payload '{"query": "Enter https://example.com in the search box and then describe the page.", "kiosk": true}' -``` +> **Note:** n2 outputs coordinates in a 1000x1000 relative space, which are scaled to the actual screen dimensions per action. -## Viewport Configuration +See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. -Yutori n1.5 recommends a **1280×800 (WXGA, 16:10)** viewport for best grounding accuracy. +## Screenshots -> **Note:** n1.5 outputs coordinates in a 1000×1000 relative space, which are automatically scaled to the actual viewport dimensions. +Screenshots are converted to WebP before they are sent. Yutori caps requests at 10 MB, and a full-screen PNG trajectory blows past that on its own. -See [Kernel Viewport Documentation](https://www.kernel.sh/docs/browsers/viewport) for all supported configurations. +The loop also drops screenshots the model will not read: n2 keeps images from only the last 2 image-bearing messages, so older ones are stripped from each request while every message's text is kept. -## Screenshots +## Tools -Screenshots are automatically converted to WebP format for better compression across multi-step trajectories, as recommended by Yutori. +This template pins the `computer_use_tools-20260825` tool set. n2 rejects `disable_tools`, so all five tools are always served and the loop answers all of them. -## n1.5-latest Supported Actions +### `computer_batch` -This template uses the `browser_tools_core-20260403` tool set — coordinate-based browser actions that operate on screenshots only. +n2 returns a whole action list in one call. The loop runs them in order, stops at the first error, and replies with a single tool result carrying one screenshot taken after the last action that ran. Every coordinate in a batch refers to the screenshot from *before* the batch started. | Action | Description | |--------|-------------| | `left_click` | Left mouse click at coordinates (supports `modifier`) | | `double_click` | Double-click at coordinates (supports `modifier`) | | `triple_click` | Triple-click at coordinates (supports `modifier`) | -| `middle_click` | Middle mouse click at coordinates | -| `right_click` | Right mouse click at coordinates | -| `mouse_move` | Move mouse to coordinates without clicking | -| `mouse_down` | Press the left mouse button at coordinates | -| `mouse_up` | Release the left mouse button at coordinates | -| `scroll` | Scroll page in a direction | -| `type` | Type text into focused element | -| `key_press` | Send a single key or key combination | -| `hold_key` | Hold a key for a duration | -| `drag` | Click-and-drag operation | -| `wait` | Pause for UI to update | -| `refresh` | Reload current page | -| `go_back` | Navigate back in history | -| `go_forward` | Navigate forward in history | -| `goto_url` | Navigate to a URL | - -### Disabled tools - -The DOM/Playwright-based "expanded" tools (`extract_elements`, `find`, `set_element_value`, `execute_js`) are intentionally disabled via the `disable_tools` request parameter — this template runs computer-use only and does not expose a Playwright page to the model. +| `middle_click` | Middle mouse click at coordinates (supports `modifier`) | +| `right_click` | Right mouse click at coordinates (supports `modifier`) | +| `scroll` | Scroll up or down at coordinates, in wheel notches | +| `type` | Type text into the focused element | +| `key_press` | Press a key, combination, or sequence | +| `drag` | Drag from `start_coordinates` to `coordinates` | +| `mouse_move` | Move the mouse to coordinates without clicking | +| `mouse_down` | Press and hold the left button (coordinates optional) | +| `mouse_up` | Release the left button (coordinates optional) | +| `hold_key` | Hold a key down for a duration | +| `wait` | Pause without interacting | +| `screenshot` | Look without acting | + +Horizontal scrolling is not available, and there are no `goto_url` / `go_back` / `go_forward` / `refresh` actions — n2 drives the address bar and browser buttons like a person would. + +### `bash`, `read`, `write`, `edit` + +n2 can also work on the browser VM directly. These map onto Kernel's process and filesystem APIs: + +| Tool | Backed by | +|------|-----------| +| `bash` | `browsers.process.exec` — each call is a separate process; the working directory carries over between calls, environment variables do not. `run_in_background` detaches the command and reports its pid and log path. | +| `read` | `browsers.fs.readFile` — returns `cat -n` output, with `offset` / `limit` paging | +| `write` | `browsers.fs.writeFile` | +| `edit` | read, exact-string replace, write. Refuses to edit a file that has not been read in this session. | ## Resources -- [Yutori n1.5 API Documentation](https://docs.yutori.com/reference/n1-5) +- [Yutori n2 API Documentation](https://docs.yutori.com/reference/n2) - [Kernel Documentation](https://www.kernel.sh/docs/quickstart) diff --git a/pkg/templates/typescript/yutori/index.ts b/pkg/templates/typescript/yutori/index.ts index 215a930b..f52fc45e 100644 --- a/pkg/templates/typescript/yutori/index.ts +++ b/pkg/templates/typescript/yutori/index.ts @@ -1,6 +1,6 @@ import { Kernel, type KernelContext } from '@onkernel/sdk'; import type OpenAI from 'openai'; -import { samplingLoop } from './loop'; +import { samplingLoop, type ReasoningEffort } from './loop'; import { KernelBrowserSession } from './session'; const kernel = new Kernel(); @@ -10,7 +10,7 @@ const app = kernel.app('ts-yutori-cua'); interface QueryInput { query: string; record_replay?: boolean; - kiosk?: boolean; + reasoning_effort?: ReasoningEffort; user_timezone?: string; user_location?: string; } @@ -35,13 +35,11 @@ app.action( throw new Error('Query is required'); } - // Create browser session with optional replay recording and kiosk mode - const kioskMode = payload.kiosk ?? false; + // Create browser session with optional replay recording const session = new KernelBrowserSession(kernel, { invocationId: ctx.invocation_id, stealth: true, recordReplay: payload.record_replay ?? false, - kioskMode, }); await session.start(); @@ -50,14 +48,16 @@ app.action( try { // Run the sampling loop const { finalAnswer, messages } = await samplingLoop({ - model: 'n1.5-latest', + model: 'n2', task: payload.query, apiKey: YUTORI_API_KEY, kernel, sessionId: session.sessionId, - viewportWidth: session.viewportWidth, - viewportHeight: session.viewportHeight, - kioskMode, + // n2 sees the whole screen, browser chrome included — the Kernel + // viewport is the window size, so it is the screen size here. + screenWidth: session.viewportWidth, + screenHeight: session.viewportHeight, + reasoningEffort: payload.reasoning_effort, userTimezone: payload.user_timezone, userLocation: payload.user_location, }); diff --git a/pkg/templates/typescript/yutori/loop.ts b/pkg/templates/typescript/yutori/loop.ts index 2b2c8c7a..b2c5c580 100644 --- a/pkg/templates/typescript/yutori/loop.ts +++ b/pkg/templates/typescript/yutori/loop.ts @@ -1,38 +1,38 @@ /** - * Yutori n1.5 Sampling Loop + * Yutori n2 Sampling Loop * - * Implements the agent loop for Yutori's n1.5-latest computer use model. - * n1.5-latest uses an OpenAI-compatible API with tool_calls: - * - Actions are returned via tool_calls in the assistant message - * - Tool results use role: "tool" with matching tool_call_id + * Implements the agent loop for Yutori's n2 computer use model. + * n2 uses an OpenAI-compatible API with tool_calls: + * - GUI actions arrive as a single `computer_batch` call holding an action list + * - `bash`, `read`, `write`, and `edit` arrive as their own calls + * - Every call needs one tool result with a matching tool_call_id * - The model stops by returning content without tool_calls * - Coordinates are returned in 1000x1000 space and need scaling * - * @see https://docs.yutori.com/reference/n1-5 + * @see https://docs.yutori.com/reference/n2 */ import OpenAI from 'openai'; import type { Kernel } from '@onkernel/sdk'; -import { ComputerTool, type N15Action, type ToolResult } from './tools/computer'; +import { ComputerTool, type BatchOutcome, type N2Action } from './tools/computer'; +import { SystemTools } from './tools/system'; -// Tools that require a Playwright page / DOM access. The default core tool set -// already excludes them, but we also list them in `disable_tools` so the -// exclusion is explicit and survives if the default ever changes. -const DISABLED_TOOLS = ['extract_elements', 'find', 'set_element_value', 'execute_js']; -const TOOL_SET = 'browser_tools_core-20260403'; +// Pin the tool set so a change to the server default can't change which tools +// this run exposes. n2 rejects `disable_tools`, so the set is served whole. +const TOOL_SET = 'computer_use_tools-20260825'; -const NAVIGATOR_COORDINATE_SCALE = 1000; +export type ReasoningEffort = 'none' | 'low' | 'medium' | 'xhigh'; -// Screenshot-trimming defaults mirror Yutori's reference loop: -// https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/payload.py -// Trimming is size-triggered — we only drop old screenshots when the payload -// exceeds MAX_REQUEST_BYTES, and we always keep at least KEEP_RECENT_SCREENSHOTS. +// Requests are capped at 10 MB. n2 also only reads images from the last 2 +// image-bearing messages, so anything older is dropped before sending rather +// than paying to upload screenshots the model will discard server-side. const MAX_REQUEST_BYTES = 9_500_000; -const KEEP_RECENT_SCREENSHOTS = 6; +const KEEP_IMAGE_MESSAGES = 2; interface YutoriExtras { tool_set: string; - disable_tools: string[]; + reasoning_effort?: ReasoningEffort; + prev_request_id?: string; } interface SamplingLoopOptions { @@ -43,9 +43,9 @@ interface SamplingLoopOptions { sessionId: string; maxCompletionTokens?: number; maxIterations?: number; - viewportWidth?: number; - viewportHeight?: number; - kioskMode?: boolean; + screenWidth?: number; + screenHeight?: number; + reasoningEffort?: ReasoningEffort; userTimezone?: string; userLocation?: string; } @@ -56,16 +56,18 @@ export interface SamplingLoopResult { } export async function samplingLoop({ - model = 'n1.5-latest', + model = 'n2', task, apiKey, kernel, sessionId, - maxCompletionTokens = 4096, + // Shared by the reasoning trace and the tool call, so a low value truncates + // the call. + maxCompletionTokens = 16384, maxIterations = 100, - viewportWidth = 1280, - viewportHeight = 800, - kioskMode = false, + screenWidth = 1280, + screenHeight = 800, + reasoningEffort, userTimezone = 'America/Los_Angeles', userLocation = 'San Francisco, CA, US', }: SamplingLoopOptions): Promise { @@ -74,80 +76,64 @@ export async function samplingLoop({ baseURL: 'https://api.yutori.com/v1', }); - const computerTool = new ComputerTool(kernel, sessionId, viewportWidth, viewportHeight, kioskMode); - - const initialScreenshot = await computerTool.screenshot(); + const computerTool = new ComputerTool(kernel, sessionId, screenWidth, screenHeight); + const systemTools = new SystemTools(kernel, sessionId); // Append location/timezone/current-date context to the task — mirrors Yutori's // format_task_with_context helper and helps the model with date-sensitive // judgments. https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/context.py - const taskWithContext = formatTaskWithContext(task, userTimezone, userLocation); - const conversationMessages: OpenAI.ChatCompletionMessageParam[] = [ { role: 'user', content: [ - { type: 'text', text: taskWithContext }, - ...(initialScreenshot.base64Image - ? [{ - type: 'image_url' as const, - image_url: { - url: `data:image/webp;base64,${initialScreenshot.base64Image}`, - }, - }] - : []), + { type: 'text', text: formatTaskWithContext(task, userTimezone, userLocation) }, + imagePart(await computerTool.screenshot()), ], }, ]; let iteration = 0; let finalAnswer: string | undefined; + let prevRequestId: string | undefined; while (iteration < maxIterations) { iteration++; console.log(`\n=== Iteration ${iteration} ===`); - const { messages: requestMessages, removed } = trimmedForRequest(conversationMessages); - if (removed > 0) { - console.log(`Trimmed ${removed} old screenshot(s) to fit request size limit`); - } + const requestMessages = trimmedForRequest(conversationMessages); + + // n2-specific knobs (not in OpenAI SDK types). The openai-node SDK + // serializes the body as-is, so these go at the top level via a spread — + // unlike the Python SDK, there is no `extra_body` kwarg here. + const yutoriExtras: YutoriExtras = { + tool_set: TOOL_SET, + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), + // Groups the trajectory into one conversation for usage reporting. + ...(prevRequestId ? { prev_request_id: prevRequestId } : {}), + }; let response; try { - // n1.5-specific knobs (not in OpenAI SDK types). The openai-node SDK - // serializes the body as-is, so these go at the top level via a spread — - // unlike the Python SDK, there is no `extra_body` kwarg here. - // tool_set selects the core (coordinate-based) tools. - // disable_tools is a defense-in-depth exclusion of DOM/Playwright tools. - const yutoriExtras: YutoriExtras = { - tool_set: TOOL_SET, - disable_tools: DISABLED_TOOLS, - }; - response = await client.chat.completions.create({ - model, - messages: requestMessages, - max_completion_tokens: maxCompletionTokens, - temperature: 0.3, - ...yutoriExtras, - }); + response = await client.chat.completions.create( + requestBody(model, requestMessages, maxCompletionTokens, yutoriExtras), + ); } catch (apiError) { console.error('API call failed:', apiError); throw apiError; } - if (!response.choices || response.choices.length === 0) { - console.error('No choices in response:', JSON.stringify(response, null, 2)); - throw new Error('No choices in API response'); - } + prevRequestId = (response as { _request_id?: string })._request_id ?? prevRequestId; - const assistantMessage = response.choices[0]?.message; + const assistantMessage = response.choices?.[0]?.message; if (!assistantMessage) { + console.error('No choices in response:', JSON.stringify(response, null, 2)); throw new Error('No response from model'); } console.log('Assistant content:', assistantMessage.content || '(none)'); - // Preserve full assistant message (including tool_calls) in history + // Push the assistant message unchanged — `reasoning_content` rides along on + // it, and n2 needs it echoed back to keep its reasoning across turns. conversationMessages.push(assistantMessage); const toolCalls = assistantMessage.tool_calls; @@ -159,67 +145,21 @@ export async function samplingLoop({ break; } - for (const toolCall of toolCalls) { - const actionName = toolCall.function.name; - let args: Record; - try { - args = JSON.parse(toolCall.function.arguments); - } catch { - console.error('Failed to parse tool_call arguments:', toolCall.function.arguments); + for (const [index, toolCall] of toolCalls.entries()) { + // n2 usually answers with one call, but `parallel_tool_calls` is pinned on + // server-side so a turn can carry several. Only the first one was planned + // against the screenshot the model actually saw, so run it and make the + // rest re-plan. + if (index > 0) { conversationMessages.push({ role: 'tool', tool_call_id: toolCall.id, - content: 'Error: failed to parse arguments', + content: 'Not executed: planned against a screenshot the previous call already changed. Re-plan from the latest screenshot.', }); continue; } - const action: N15Action = { - action_type: actionName as N15Action['action_type'], - ...args, - }; - - console.log('Executing action:', actionName, args); - - const scaledAction = scaleCoordinates(action, viewportWidth, viewportHeight); - - let result: ToolResult; - try { - result = await computerTool.execute(scaledAction); - } catch (error) { - console.error('Action failed:', error); - result = { - error: error instanceof Error ? error.message : String(error), - }; - } - - if (result.base64Image) { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - // Yutori n1 accepts image content arrays in tool messages (not yet in OpenAI SDK types) - content: [ - { - type: 'image_url', - image_url: { - url: `data:image/webp;base64,${result.base64Image}`, - }, - }, - ] as unknown as string, - }); - } else if (result.error) { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: `Action failed: ${result.error}`, - }); - } else { - conversationMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: result.output || 'OK', - }); - } + conversationMessages.push(await runToolCall(toolCall, computerTool, systemTools)); } } @@ -229,27 +169,20 @@ export async function samplingLoop({ if (iteration >= maxIterations && !finalAnswer) { console.log('Max iterations reached — requesting summary'); try { - const finalScreenshot = await computerTool.screenshot(); conversationMessages.push({ role: 'user', content: [ { type: 'text', text: formatStopAndSummarize(task) }, - ...(finalScreenshot.base64Image - ? [{ - type: 'image_url' as const, - image_url: { url: `data:image/webp;base64,${finalScreenshot.base64Image}` }, - }] - : []), + imagePart(await computerTool.screenshot()), ], }); - const { messages: summaryMessages } = trimmedForRequest(conversationMessages); - const summaryResponse = await client.chat.completions.create({ - model, - messages: summaryMessages, - max_completion_tokens: maxCompletionTokens, - temperature: 0.3, - ...({ tool_set: TOOL_SET, disable_tools: DISABLED_TOOLS } satisfies YutoriExtras), - }); + + const summaryResponse = await client.chat.completions.create( + requestBody(model, trimmedForRequest(conversationMessages), maxCompletionTokens, { + tool_set: TOOL_SET, + }), + ); + const summary = summaryResponse.choices[0]?.message; if (summary) { conversationMessages.push(summary); @@ -266,6 +199,80 @@ export async function samplingLoop({ }; } +// n2's knobs are not in the OpenAI SDK types — the openai-node SDK serializes +// the body as-is, so they ride at the top level. `reasoning_effort` collides +// with OpenAI's own narrower enum, hence the cast on the way out. +function requestBody( + model: string, + messages: OpenAI.ChatCompletionMessageParam[], + maxCompletionTokens: number, + extras: YutoriExtras, +): OpenAI.ChatCompletionCreateParamsNonStreaming { + return { + model, + messages, + max_completion_tokens: maxCompletionTokens, + temperature: 0.6, + ...extras, + } as OpenAI.ChatCompletionCreateParamsNonStreaming; +} + +async function runToolCall( + toolCall: OpenAI.ChatCompletionMessageToolCall, + computerTool: ComputerTool, + systemTools: SystemTools, +): Promise { + const name = toolCall.function.name; + + let args: Record; + try { + args = JSON.parse(toolCall.function.arguments); + } catch { + console.error('Failed to parse tool_call arguments:', toolCall.function.arguments); + return { role: 'tool', tool_call_id: toolCall.id, content: 'Error: failed to parse arguments' }; + } + + console.log('Executing tool:', name, JSON.stringify(args)); + + if (name === 'computer_batch') { + const outcome = await computerTool.runBatch((args.actions ?? []) as N2Action[]); + + return { + role: 'tool', + tool_call_id: toolCall.id, + // n2 accepts image content arrays in tool messages (not yet in OpenAI SDK types) + content: [ + { type: 'text', text: describeOutcome(outcome) }, + imagePart(await computerTool.screenshot()), + ] as unknown as string, + }; + } + + try { + return { role: 'tool', tool_call_id: toolCall.id, content: await systemTools.execute(name, args) }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`${name} failed:`, message); + return { role: 'tool', tool_call_id: toolCall.id, content: `${name} failed: ${message}` }; + } +} + +function describeOutcome({ executed, total, failure }: BatchOutcome): string { + if (!failure) { + return `Executed ${executed} of ${total} actions.`; + } + + return ( + `Executed ${executed} of ${total} actions. ` + + `Action ${failure.index + 1} (${failure.name}) failed: ${failure.message}. ` + + `The remaining actions were skipped.` + ); +} + +function imagePart(base64Image: string): OpenAI.ChatCompletionContentPartImage { + return { type: 'image_url', image_url: { url: `data:image/webp;base64,${base64Image}` } }; +} + function formatTaskWithContext(task: string, userTimezone: string, userLocation: string): string { const now = new Date(); const tzLabel = resolveTimezone(userTimezone); @@ -319,30 +326,6 @@ function formatStopAndSummarize(task: string): string { ); } -function scaleCoordinates(action: N15Action, viewportWidth: number, viewportHeight: number): N15Action { - const scaled = { ...action }; - - if (scaled.coordinates) { - scaled.coordinates = denormalize(scaled.coordinates, viewportWidth, viewportHeight); - } - - if (scaled.start_coordinates) { - scaled.start_coordinates = denormalize(scaled.start_coordinates, viewportWidth, viewportHeight); - } - - return scaled; -} - -// Map [0, 1000] coordinates into viewport pixels and clamp to [0, dim-1] so a -// boundary value like 1000 doesn't land one pixel outside the viewport. -function denormalize(coords: [number, number], width: number, height: number): [number, number] { - const rawX = Math.round((coords[0] / NAVIGATOR_COORDINATE_SCALE) * width); - const rawY = Math.round((coords[1] / NAVIGATOR_COORDINATE_SCALE) * height); - const x = Math.max(0, Math.min(width - 1, rawX)); - const y = Math.max(0, Math.min(height - 1, rawY)); - return [x, y]; -} - interface ImagePart { type: 'image_url'; image_url: { url: string }; @@ -365,70 +348,54 @@ function messageHasImage(msg: OpenAI.ChatCompletionMessageParam): boolean { return content.some((p) => typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'image_url'); } -function stripOneImage(msg: OpenAI.ChatCompletionMessageParam): boolean { +function stripImages(msg: OpenAI.ChatCompletionMessageParam): void { const content = (msg as { content?: unknown }).content; - if (!Array.isArray(content)) return false; + if (!Array.isArray(content)) return; - let removed = false; - const next: ContentPart[] = []; - for (const part of content as ContentPart[]) { - if (!removed && typeof part === 'object' && part !== null && (part as { type?: unknown }).type === 'image_url') { - removed = true; - continue; - } - next.push(part); - } - if (!removed) return false; + const next = (content as ContentPart[]).filter( + (p) => !(typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'image_url'), + ); const hasText = next.some((p) => typeof p === 'object' && p !== null && (p as { type?: unknown }).type === 'text'); if (!hasText) { - next.push({ type: 'text', text: 'Screenshot omitted to stay under request size limit.' }); + next.push({ type: 'text', text: 'Screenshot omitted — superseded by a newer one.' }); } (msg as { content: unknown }).content = next; - return true; } +/** + * Drop the screenshots n2 will not look at, keeping the text of every turn. + * + * The model reads images from the last KEEP_IMAGE_MESSAGES image-bearing + * messages and ignores the rest, so those are stripped every request. If the + * payload is still over the cap after that, the older of the kept screenshots + * goes too — the latest one is always kept, since a request with no screenshot + * badly degrades grounding. + */ function trimmedForRequest( messages: OpenAI.ChatCompletionMessageParam[], -): { messages: OpenAI.ChatCompletionMessageParam[]; removed: number } { +): OpenAI.ChatCompletionMessageParam[] { // Deep-copy so the caller's full history is preserved unchanged. const trimmed = JSON.parse(JSON.stringify(messages)) as OpenAI.ChatCompletionMessageParam[]; - let size = estimateSize(trimmed); - if (size <= MAX_REQUEST_BYTES) return { messages: trimmed, removed: 0 }; - const imageIndices: number[] = []; for (let i = 0; i < trimmed.length; i++) { if (messageHasImage(trimmed[i]!)) imageIndices.push(i); } - if (imageIndices.length === 0) return { messages: trimmed, removed: 0 }; - - const keep = Math.max(1, KEEP_RECENT_SCREENSHOTS); - const protectedIdx = new Set(imageIndices.slice(-keep)); - let removed = 0; - - for (const idx of imageIndices) { - if (size <= MAX_REQUEST_BYTES) break; - if (protectedIdx.has(idx)) continue; - if (stripOneImage(trimmed[idx]!)) { - removed++; - size = estimateSize(trimmed); - } + + for (const idx of imageIndices.slice(0, -KEEP_IMAGE_MESSAGES)) { + stripImages(trimmed[idx]!); } - // If still over, strip from the protected window too — but always keep the latest. - if (size > MAX_REQUEST_BYTES) { - const lastIdx = imageIndices[imageIndices.length - 1]!; - for (const idx of imageIndices) { - if (size <= MAX_REQUEST_BYTES) break; - if (idx === lastIdx) continue; - if (stripOneImage(trimmed[idx]!)) { - removed++; - size = estimateSize(trimmed); - } - } + // The latest screenshot is never dropped — a request without one badly + // degrades grounding — so the previous one is the only thing left to give up. + const kept = imageIndices.slice(-KEEP_IMAGE_MESSAGES); + for (const idx of kept.slice(0, -1)) { + if (estimateSize(trimmed) <= MAX_REQUEST_BYTES) break; + console.warn('Payload still over the 10 MB cap — dropping the previous screenshot too'); + stripImages(trimmed[idx]!); } - return { messages: trimmed, removed }; + return trimmed; } diff --git a/pkg/templates/typescript/yutori/session.ts b/pkg/templates/typescript/yutori/session.ts index b0edcc3d..47185f2f 100644 --- a/pkg/templates/typescript/yutori/session.ts +++ b/pkg/templates/typescript/yutori/session.ts @@ -22,8 +22,6 @@ export interface SessionOptions { viewportWidth?: number; /** Viewport height */ viewportHeight?: number; - /** Launch browser in kiosk mode (hides address bar and tabs) */ - kioskMode?: boolean; } export interface SessionInfo { @@ -45,7 +43,6 @@ const DEFAULT_OPTIONS: Required> = { replayGracePeriod: 5.0, viewportWidth: 1280, viewportHeight: 800, - kioskMode: false, }; /** @@ -105,10 +102,6 @@ export class KernelBrowserSession { return this.options.viewportHeight; } - get kioskMode(): boolean { - return this.options.kioskMode; - } - get info(): SessionInfo { return { sessionId: this.sessionId, @@ -130,7 +123,6 @@ export class KernelBrowserSession { width: this.options.viewportWidth, height: this.options.viewportHeight, }, - kiosk_mode: this.options.kioskMode, }); this._sessionId = browser.session_id ?? null; diff --git a/pkg/templates/typescript/yutori/tools/computer.ts b/pkg/templates/typescript/yutori/tools/computer.ts index a201e009..431402da 100644 --- a/pkg/templates/typescript/yutori/tools/computer.ts +++ b/pkg/templates/typescript/yutori/tools/computer.ts @@ -1,10 +1,12 @@ /** - * Yutori n1.5 Computer Tool + * Yutori n2 Computer Tool * - * Maps n1.5-latest action format to Kernel's Computer Controls API. - * Screenshots are converted to WebP for better compression across multi-step trajectories. + * Maps n2's `computer_batch` action list onto Kernel's Computer Controls API. + * n2 plans every action in a batch against the screenshot taken *before* the + * batch ran, so the batch executes sequentially, stops at the first error, and + * answers with a single screenshot taken after the last action that ran. * - * @see https://docs.yutori.com/reference/n1-5 + * @see https://docs.yutori.com/reference/n2 */ import { Buffer } from 'buffer'; @@ -12,27 +14,24 @@ import type { Kernel } from '@onkernel/sdk'; import sharp from 'sharp'; const TYPING_DELAY_MS = 12; -const SCREENSHOT_DELAY_MS = 150; -const ACTION_DELAY_MS = 300; +// Let the UI settle between actions in a batch, and again before the screenshot +// that the model will plan its next batch against. +const INTER_ACTION_DELAY_MS = 80; +const SETTLE_DELAY_MS = 400; -// n1.5 scroll `amount` is in "wheel units" where 1 unit ≈ 10% of the viewport -// height (~80px at 800px tall). Kernel's `delta_y` is a wheel-event repeat -// count where each tick is much smaller in practice, so we multiply. -const SCROLL_NOTCHES_PER_AMOUNT = 4; +const NAVIGATOR_COORDINATE_SCALE = 1000; + +// n2's scroll `amount` is a wheel notch count (1-50), which is exactly what +// Kernel's delta_x/delta_y take ("xdotool wheel units"), so it passes through. +const DEFAULT_SCROLL_AMOUNT = 3; // WebP quality for screenshots. Kernel returns PNGs, which are crisp and // tolerate aggressive WebP compression with no visible degradation — matches // Yutori SDK's DEFAULT_WEBP_QUALITY_FOR_PNG=30 (yutori-sdk-python/yutori/ -// navigator/images.py). Lower values cut payload size substantially on long -// multi-step trajectories. +// navigator/images.py). Requests are capped at 10 MB and a trajectory carries +// several full-screen captures, so the compression matters. const WEBP_QUALITY = 30; -export interface ToolResult { - base64Image?: string; - output?: string; - error?: string; -} - export class ToolError extends Error { constructor(message: string) { super(message); @@ -40,44 +39,52 @@ export class ToolError extends Error { } } -export type N15ActionType = +export type N2ActionName = | 'left_click' | 'double_click' | 'triple_click' | 'middle_click' | 'right_click' - | 'mouse_move' - | 'mouse_down' - | 'mouse_up' | 'scroll' | 'type' | 'key_press' - | 'hold_key' | 'drag' + | 'mouse_move' + | 'mouse_down' + | 'mouse_up' + | 'hold_key' | 'wait' - | 'refresh' - | 'go_back' - | 'go_forward' - | 'goto_url'; + | 'screenshot'; -export interface N15Action { - action_type: N15ActionType; +export interface N2ActionArgs { coordinates?: [number, number]; start_coordinates?: [number, number]; - direction?: 'up' | 'down' | 'left' | 'right'; + direction?: 'up' | 'down'; amount?: number; text?: string; key?: string; modifier?: string; duration?: number; - url?: string; } -// n1.5 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). +/** One `{name, arguments}` member of a `computer_batch` call. */ +export interface N2Action { + name: N2ActionName; + arguments?: N2ActionArgs; +} + +export interface BatchOutcome { + executed: number; + total: number; + /** Set when an action threw; the remaining actions were skipped. */ + failure?: { index: number; name: string; message: string }; +} + +// n2 emits lowercase key names (e.g. `enter`, `ctrl+c`, `down down down enter`). // Kernel's press_key expects XKeysym names (e.g. `Return`, `Ctrl`, `Page_Up`). // This map covers every key Yutori documents at -// https://docs.yutori.com/reference/n1-5#key-space — keys not in the map pass -// through unchanged (printable characters like `a`, `1`, `,` are already XKeysym). +// https://docs.yutori.com/reference/n2#key-space — keys not in the map pass +// through unchanged (printable characters like `a` and `1` are already XKeysym). // // Sister implementation (Playwright target instead of XKeysym): // https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/keys.py @@ -119,6 +126,19 @@ const KEY_MAP: Record = { // Function keys f1: 'F1', f2: 'F2', f3: 'F3', f4: 'F4', f5: 'F5', f6: 'F6', f7: 'F7', f8: 'F8', f9: 'F9', f10: 'F10', f11: 'F11', f12: 'F12', + // Punctuation — n2 sends word forms, most of which are already XKeysym names + minus: 'minus', + plus: 'plus', + equal: 'equal', + comma: 'comma', + period: 'period', + slash: 'slash', + backslash: 'backslash', + semicolon: 'semicolon', + quote: 'apostrophe', + backquote: 'grave', + bracketleft: 'bracketleft', + bracketright: 'bracketright', // Locks / special capslock: 'Caps_Lock', numlock: 'Num_Lock', @@ -133,14 +153,9 @@ function mapToken(token: string): string { return KEY_MAP[lower] ?? token.trim(); } -function normalizeUrl(url: string): string { - const trimmed = url.trim(); - return trimmed.includes('://') ? trimmed : `https://${trimmed}`; -} - -// Parse an n1.5 key expression into one Kernel combo string per sequential -// press. Spaces separate sequential presses; `+` separates simultaneous tokens -// within a press. Examples: +// Parse an n2 key expression into one Kernel combo string per sequential press. +// Spaces separate sequential presses; `+` separates simultaneous tokens within a +// press. Examples: // "enter" -> ["Return"] // "ctrl+c" -> ["Ctrl+c"] // "down down enter" -> ["Down", "Down", "Return"] @@ -156,327 +171,226 @@ function parseKeyExpression(expr: string): string[] { export class ComputerTool { private kernel: Kernel; private sessionId: string; - private width: number; - private height: number; - private kioskMode: boolean; + private screenWidth: number; + private screenHeight: number; - constructor(kernel: Kernel, sessionId: string, width = 1280, height = 800, kioskMode = false) { + constructor(kernel: Kernel, sessionId: string, screenWidth = 1280, screenHeight = 800) { this.kernel = kernel; this.sessionId = sessionId; - this.width = width; - this.height = height; - this.kioskMode = kioskMode; + this.screenWidth = screenWidth; + this.screenHeight = screenHeight; + } + + /** + * Run a `computer_batch` action list in order, stopping at the first failure. + * The caller reports the outcome and a single post-batch screenshot back to n2. + */ + async runBatch(actions: N2Action[]): Promise { + for (let i = 0; i < actions.length; i++) { + const action = actions[i]!; + try { + await this.runAction(action); + } catch (error) { + return { + executed: i, + total: actions.length, + failure: { + index: i, + name: action.name, + message: error instanceof Error ? error.message : String(error), + }, + }; + } + if (i < actions.length - 1) { + await this.sleep(INTER_ACTION_DELAY_MS); + } + } + + return { executed: actions.length, total: actions.length }; } - async execute(action: N15Action): Promise { - const { action_type } = action; + private async runAction(action: N2Action): Promise { + const args = action.arguments ?? {}; - switch (action_type) { + switch (action.name) { case 'left_click': - return this.handleClick(action, 'left', 1); + return this.click(args, 'left', 1); case 'double_click': - return this.handleClick(action, 'left', 2); + return this.click(args, 'left', 2); case 'triple_click': - return this.handleClick(action, 'left', 3); + return this.click(args, 'left', 3); case 'middle_click': - return this.handleClick(action, 'middle', 1); + return this.click(args, 'middle', 1); case 'right_click': - return this.handleClick(action, 'right', 1); - case 'mouse_move': - return this.handleMouseMove(action); - case 'mouse_down': - return this.handleMouseButton(action, 'down'); - case 'mouse_up': - return this.handleMouseButton(action, 'up'); + return this.click(args, 'right', 1); case 'scroll': - return this.handleScroll(action); + return this.scroll(args); case 'type': - return this.handleType(action); + return this.type(args); case 'key_press': - return this.handleKeyPress(action); - case 'hold_key': - return this.handleHoldKey(action); + return this.keyPress(args); case 'drag': - return this.handleDrag(action); + return this.drag(args); + case 'mouse_move': + return this.mouseMove(args); + case 'mouse_down': + return this.mouseButton(args, 'down'); + case 'mouse_up': + return this.mouseButton(args, 'up'); + case 'hold_key': + return this.holdKey(args); case 'wait': - return this.handleWait(action); - case 'refresh': - return this.handleRefresh(); - case 'go_back': - return this.handleGoBack(); - case 'go_forward': - return this.handleGoForward(); - case 'goto_url': - return this.handleGotoUrl(action); + return this.sleep(durationMs(args.duration, 2000)); + case 'screenshot': + // A batch already answers with a screenshot taken after its last action, + // so an explicit `screenshot` member has nothing to do. + return; default: - throw new ToolError(`Unknown action type: ${action_type}`); + throw new ToolError(`Unknown action: ${action.name}`); } } - private async handleClick(action: N15Action, button: 'left' | 'right' | 'middle', numClicks: number): Promise { - const coords = this.getCoordinates(action.coordinates); - const holdKeys = action.modifier ? [mapToken(action.modifier)] : undefined; + private async click(args: N2ActionArgs, button: 'left' | 'right' | 'middle', numClicks: number): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + const holdKeys = args.modifier ? [mapToken(args.modifier)] : undefined; await this.kernel.browsers.computer.clickMouse(this.sessionId, { - x: coords.x, - y: coords.y, + x, + y, button, click_type: 'click', num_clicks: numClicks, ...(holdKeys ? { hold_keys: holdKeys } : {}), }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleMouseMove(action: N15Action): Promise { - const coords = this.getCoordinates(action.coordinates); - - await this.kernel.browsers.computer.moveMouse(this.sessionId, { - x: coords.x, - y: coords.y, - }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); + private async mouseMove(args: N2ActionArgs): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + await this.kernel.browsers.computer.moveMouse(this.sessionId, { x, y }); } - private async handleMouseButton(action: N15Action, clickType: 'down' | 'up'): Promise { - const coords = this.getCoordinates(action.coordinates); + private async mouseButton(args: N2ActionArgs, clickType: 'down' | 'up'): Promise { + // Coordinates are optional here — without them the button is pressed or + // released wherever the cursor already is, which is what a manual + // mouse_move -> mouse_down -> mouse_move -> mouse_up drag relies on. + const { x, y } = args.coordinates + ? this.requireCoordinates(args.coordinates) + : await this.kernel.browsers.computer.getMousePosition(this.sessionId); await this.kernel.browsers.computer.clickMouse(this.sessionId, { - x: coords.x, - y: coords.y, + x, + y, button: 'left', click_type: clickType, }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleScroll(action: N15Action): Promise { - const coords = this.getCoordinates(action.coordinates); - const direction = action.direction; - const amount = Math.max(action.amount ?? 3, 1); + private async scroll(args: N2ActionArgs): Promise { + const { x, y } = this.requireCoordinates(args.coordinates); + const direction = args.direction; - if (!direction || !['up', 'down', 'left', 'right'].includes(direction)) { + // n2 only scrolls vertically. + if (direction !== 'up' && direction !== 'down') { throw new ToolError(`Invalid scroll direction: ${direction}`); } - // Yutori 1 unit ≈ 10% of viewport height; scale into Kernel wheel-event ticks. - const ticks = amount * SCROLL_NOTCHES_PER_AMOUNT; - - let delta_x = 0; - let delta_y = 0; - - switch (direction) { - case 'up': - delta_y = -ticks; - break; - case 'down': - delta_y = ticks; - break; - case 'left': - delta_x = -ticks; - break; - case 'right': - delta_x = ticks; - break; - } - - const holdKeys = action.modifier ? [mapToken(action.modifier)] : undefined; + const notches = Math.max(1, Math.round(args.amount ?? DEFAULT_SCROLL_AMOUNT)); + const holdKeys = args.modifier ? [mapToken(args.modifier)] : undefined; await this.kernel.browsers.computer.scroll(this.sessionId, { - x: coords.x, - y: coords.y, - delta_x, - delta_y, + x, + y, + delta_x: 0, + delta_y: direction === 'up' ? -notches : notches, ...(holdKeys ? { hold_keys: holdKeys } : {}), }); - - await this.sleep(SCREENSHOT_DELAY_MS); - const screenshotResult = await this.screenshot(); - return { - ...screenshotResult, - output: `Scrolled ${amount} unit(s) ${direction}.`, - }; } - private async handleType(action: N15Action): Promise { - const text = action.text; - if (!text) { - throw new ToolError('text is required for type action'); + private async type(args: N2ActionArgs): Promise { + if (!args.text) { + throw new ToolError('text is required for type'); } await this.kernel.browsers.computer.typeText(this.sessionId, { - text, + text: args.text, delay: TYPING_DELAY_MS, }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleKeyPress(action: N15Action): Promise { - const key = action.key; - if (!key) { - throw new ToolError('key is required for key_press action'); + private async keyPress(args: N2ActionArgs): Promise { + if (!args.key) { + throw new ToolError('key is required for key_press'); } - // n1.5 supports sequential presses ("down down down enter") — issue each - // combo as its own pressKey so they're seen as separate keystrokes. - const combos = parseKeyExpression(key); - for (const combo of combos) { + // n2 supports sequential presses ("down down down enter") — issue each combo + // as its own pressKey so they're seen as separate keystrokes. + for (const combo of parseKeyExpression(args.key)) { await this.kernel.browsers.computer.pressKey(this.sessionId, { keys: [combo] }); } - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleHoldKey(action: N15Action): Promise { - const key = action.key; - if (!key) { - throw new ToolError('key is required for hold_key action'); + private async holdKey(args: N2ActionArgs): Promise { + if (!args.key) { + throw new ToolError('key is required for hold_key'); } - // Yutori emits `duration` in seconds; Kernel SDK's pressKey takes ms. - const durationMs = action.duration && action.duration > 0 ? Math.round(action.duration * 1000) : 1000; - - const combos = parseKeyExpression(key); - for (const combo of combos) { + for (const combo of parseKeyExpression(args.key)) { await this.kernel.browsers.computer.pressKey(this.sessionId, { keys: [combo], - duration: durationMs, + duration: durationMs(args.duration, 1000), }); } - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleDrag(action: N15Action): Promise { - const startCoords = this.getCoordinates(action.start_coordinates); - const endCoords = this.getCoordinates(action.coordinates); + private async drag(args: N2ActionArgs): Promise { + const start = this.requireCoordinates(args.start_coordinates); + const end = this.requireCoordinates(args.coordinates); await this.kernel.browsers.computer.dragMouse(this.sessionId, { - path: [[startCoords.x, startCoords.y], [endCoords.x, endCoords.y]], + path: [[start.x, start.y], [end.x, end.y]], button: 'left', }); - - await this.sleep(SCREENSHOT_DELAY_MS); - return this.screenshot(); } - private async handleWait(action: N15Action): Promise { - // Yutori emits `duration` in seconds (matches reference impl). - const durationMs = action.duration && action.duration > 0 ? Math.round(action.duration * 1000) : 2000; - await this.sleep(durationMs); - return this.screenshot(); - } + /** Capture the whole screen — browser chrome included — as base64 WebP. */ + async screenshot(): Promise { + await this.sleep(SETTLE_DELAY_MS); - private async handleRefresh(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['F5'], - }); + const response = await this.kernel.browsers.computer.captureScreenshot(this.sessionId); + const pngBuffer = Buffer.from(await (await response.blob()).arrayBuffer()); + const webpBuffer = await sharp(pngBuffer).webp({ quality: WEBP_QUALITY }).toBuffer(); - await this.sleep(2000); - return this.screenshot(); + return webpBuffer.toString('base64'); } - private async handleGoBack(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Alt+Left'], - }); - - await this.sleep(1500); - return this.screenshot(); - } - - private async handleGoForward(): Promise { - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Alt+Right'], - }); - - await this.sleep(1500); - return this.screenshot(); - } - - private async handleGotoUrl(action: N15Action): Promise { - const url = action.url; - if (!url) { - throw new ToolError('url is required for goto_url action'); - } - const targetUrl = normalizeUrl(url); - - if (this.kioskMode) { - const response = await this.kernel.browsers.playwright.execute(this.sessionId, { - code: `await page.goto(${JSON.stringify(targetUrl)});`, - timeout_sec: 60, - }); - if (!response.success) { - throw new ToolError(response.error ?? 'Playwright goto failed'); - } - await this.sleep(ACTION_DELAY_MS); - return this.screenshot(); - } - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Ctrl+l'], - }); - await this.sleep(ACTION_DELAY_MS); - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Ctrl+a'], - }); - await this.sleep(100); - - await this.kernel.browsers.computer.typeText(this.sessionId, { - text: targetUrl, - delay: TYPING_DELAY_MS, - }); - await this.sleep(ACTION_DELAY_MS); - - await this.kernel.browsers.computer.pressKey(this.sessionId, { - keys: ['Return'], - }); - - await this.sleep(2000); - return this.screenshot(); - } - - async screenshot(): Promise { - try { - const response = await this.kernel.browsers.computer.captureScreenshot(this.sessionId); - const blob = await response.blob(); - const arrayBuffer = await blob.arrayBuffer(); - const pngBuffer = Buffer.from(arrayBuffer); - const webpBuffer = await sharp(pngBuffer).webp({ quality: WEBP_QUALITY }).toBuffer(); - - return { - base64Image: webpBuffer.toString('base64'), - }; - } catch (error) { - throw new ToolError(`Failed to take screenshot: ${error}`); - } - } - - private getCoordinates(coords?: [number, number]): { x: number; y: number } { + // Map [0, 1000] coordinates into screen pixels and clamp to [0, dim-1] so a + // boundary value like 1000 doesn't land one pixel outside the screen. + private requireCoordinates(coords?: [number, number]): { x: number; y: number } { if (!coords || coords.length !== 2) { - return { x: Math.floor(this.width / 2), y: Math.floor(this.height / 2) }; + throw new ToolError(`coordinates are required, got ${JSON.stringify(coords)}`); } - const [x, y] = coords; - if (typeof x !== 'number' || typeof y !== 'number' || x < 0 || y < 0) { + const [nx, ny] = coords; + if (typeof nx !== 'number' || typeof ny !== 'number') { throw new ToolError(`Invalid coordinates: ${JSON.stringify(coords)}`); } - return { x, y }; + return { + x: clamp(Math.round((nx / NAVIGATOR_COORDINATE_SCALE) * this.screenWidth), this.screenWidth), + y: clamp(Math.round((ny / NAVIGATOR_COORDINATE_SCALE) * this.screenHeight), this.screenHeight), + }; } private sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } } + +function clamp(value: number, dimension: number): number { + return Math.max(0, Math.min(dimension - 1, value)); +} + +// n2 emits `duration` in seconds; Kernel and setTimeout take milliseconds. +function durationMs(duration: number | undefined, fallbackMs: number): number { + return duration && duration > 0 ? Math.round(duration * 1000) : fallbackMs; +} diff --git a/pkg/templates/typescript/yutori/tools/system.ts b/pkg/templates/typescript/yutori/tools/system.ts new file mode 100644 index 00000000..796bc362 --- /dev/null +++ b/pkg/templates/typescript/yutori/tools/system.ts @@ -0,0 +1,225 @@ +/** + * Yutori n2 Shell and File Tools + * + * n2's tool set always ships `bash`, `read`, `write`, and `edit` alongside + * `computer_batch` — `disable_tools` is rejected by the API, so a loop has to + * answer these too. They run against the Kernel browser VM via the process and + * filesystem APIs. + * + * @see https://docs.yutori.com/reference/n2 + */ + +import { Buffer } from 'buffer'; +import type { Kernel } from '@onkernel/sdk'; +import { ToolError } from './computer'; + +const DEFAULT_TIMEOUT_SEC = 120; +const MAX_TIMEOUT_SEC = 600; +const DEFAULT_READ_LIMIT = 2000; +const MAX_WRITE_CHARS = 256_000; + +// bash: "The working directory persists across calls; environment variables and +// shell functions do not." Each exec is its own process, so the command reports +// its final directory on a sentinel line that we strip before returning stdout. +const CWD_SENTINEL = '__n2_cwd__'; + +interface BashArgs { + command?: string; + timeout?: number; + run_in_background?: boolean; +} + +interface ReadArgs { + file_path?: string; + offset?: number; + limit?: number; +} + +interface WriteArgs { + file_path?: string; + content?: string; +} + +interface EditArgs { + file_path?: string; + old_string?: string; + new_string?: string; + replace_all?: boolean; +} + +export class SystemTools { + private kernel: Kernel; + private sessionId: string; + private cwd: string | undefined; + private seenPaths = new Set(); + + constructor(kernel: Kernel, sessionId: string) { + this.kernel = kernel; + this.sessionId = sessionId; + } + + async execute(name: string, args: Record): Promise { + switch (name) { + case 'bash': + return this.bash(args as BashArgs); + case 'read': + return this.read(args as ReadArgs); + case 'write': + return this.write(args as WriteArgs); + case 'edit': + return this.edit(args as EditArgs); + default: + throw new ToolError(`Unknown tool: ${name}`); + } + } + + private async bash(args: BashArgs): Promise { + const command = args.command; + if (!command) { + throw new ToolError('command is required for bash'); + } + + const timeoutSec = Math.min(args.timeout ?? DEFAULT_TIMEOUT_SEC, MAX_TIMEOUT_SEC); + + if (args.run_in_background) { + return this.bashBackground(command); + } + + const script = [ + command, + 'n2_status=$?', + `printf '\\n${CWD_SENTINEL}%s' "$(pwd)"`, + 'exit $n2_status', + ].join('\n'); + + const result = await this.kernel.browsers.process.exec(this.sessionId, { + command: 'bash', + args: ['-lc', script], + ...(this.cwd ? { cwd: this.cwd } : {}), + timeout_sec: timeoutSec, + }); + + const stdout = this.takeCwd(decode(result.stdout_b64)); + const stderr = decode(result.stderr_b64); + + return formatCommandResult(stdout, stderr, result.exit_code); + } + + private async read(args: ReadArgs): Promise { + const filePath = args.file_path; + if (!filePath) { + throw new ToolError('file_path is required for read'); + } + + const response = await this.kernel.browsers.fs.readFile(this.sessionId, { path: filePath }); + const lines = (await response.text()).split('\n'); + + const offset = Math.max(0, args.offset ?? 0); + const limit = Math.max(1, args.limit ?? DEFAULT_READ_LIMIT); + const page = lines.slice(offset, offset + limit); + + this.seenPaths.add(filePath); + + if (page.length === 0) { + return `${filePath} has ${lines.length} line(s); offset ${offset} is past the end.`; + } + + // cat -n format, so line numbers survive into the model's next edit. + return page.map((line, i) => `${String(offset + i + 1).padStart(6)}\t${line}`).join('\n'); + } + + private async write(args: WriteArgs): Promise { + const filePath = args.file_path; + const content = args.content; + if (!filePath || content === undefined) { + throw new ToolError('file_path and content are required for write'); + } + if (content.length > MAX_WRITE_CHARS) { + throw new ToolError(`content exceeds the ${MAX_WRITE_CHARS} character cap`); + } + + await this.kernel.browsers.fs.writeFile(this.sessionId, content, { path: filePath }); + this.seenPaths.add(filePath); + + return `Wrote ${content.length} character(s) to ${filePath}.`; + } + + private async edit(args: EditArgs): Promise { + const { file_path: filePath, old_string: oldString, new_string: newString } = args; + if (!filePath || oldString === undefined || newString === undefined) { + throw new ToolError('file_path, old_string, and new_string are required for edit'); + } + // n2 is expected to know the current bytes before changing them. + if (!this.seenPaths.has(filePath)) { + throw new ToolError(`${filePath} has not been read in this session — read it before editing.`); + } + + const response = await this.kernel.browsers.fs.readFile(this.sessionId, { path: filePath }); + const original = await response.text(); + + const occurrences = original.split(oldString).length - 1; + if (occurrences === 0) { + throw new ToolError(`old_string not found in ${filePath}`); + } + if (occurrences > 1 && !args.replace_all) { + throw new ToolError( + `old_string matches ${occurrences} times in ${filePath} — pass replace_all or include more context.`, + ); + } + + const updated = args.replace_all + ? original.split(oldString).join(newString) + : original.replace(oldString, newString); + + await this.kernel.browsers.fs.writeFile(this.sessionId, updated, { path: filePath }); + + return `Replaced ${args.replace_all ? occurrences : 1} occurrence(s) in ${filePath}.`; + } + + private async bashBackground(command: string): Promise { + const logPath = `/tmp/n2-bg-${Date.now()}.log`; + const script = `nohup bash -c ${shellQuote(command)} > ${logPath} 2>&1 &\necho $!`; + + const result = await this.kernel.browsers.process.exec(this.sessionId, { + command: 'bash', + args: ['-lc', script], + ...(this.cwd ? { cwd: this.cwd } : {}), + }); + + const pid = decode(result.stdout_b64).trim(); + + return [ + `Started in the background with pid ${pid}.`, + `Output is being written to ${logPath} — use the read tool to check on it.`, + `Cancel it with: kill ${pid}`, + ].join('\n'); + } + + /** Strip the trailing sentinel line and remember the directory it reported. */ + private takeCwd(stdout: string): string { + const marker = stdout.lastIndexOf(`\n${CWD_SENTINEL}`); + if (marker === -1) { + return stdout; + } + + this.cwd = stdout.slice(marker + CWD_SENTINEL.length + 1).trim() || this.cwd; + return stdout.slice(0, marker); + } +} + +function decode(base64?: string): string { + return base64 ? Buffer.from(base64, 'base64').toString('utf-8') : ''; +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +function formatCommandResult(stdout: string, stderr: string, exitCode?: number): string { + const parts: string[] = []; + if (stdout.trim()) parts.push(stdout.trimEnd()); + if (stderr.trim()) parts.push(`stderr:\n${stderr.trimEnd()}`); + if (exitCode) parts.push(`Exited with code ${exitCode}.`); + + return parts.length > 0 ? parts.join('\n') : 'Command produced no output.'; +} From cf32041014d319fc3f7d7d67c8da40a0b39814b5 Mon Sep 17 00:00:00 2001 From: dprevoznik <58714078+dprevoznik@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:39:39 +0000 Subject: [PATCH 2/2] templates: fix dollar-token expansion in the n2 edit tool String.prototype.replace with a string replacement expands $$, $&, $\` and $' inside the model's new_string, corrupting shell and script edits. Use the replacer function form. Python's str.replace has no such behavior. Also pass cwd explicitly in the TypeScript bash tool so both adapters send the same field; the Kernel API types it as nullable and accepts null. Co-Authored-By: Claude Opus 5 --- pkg/templates/typescript/yutori/tools/system.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/templates/typescript/yutori/tools/system.ts b/pkg/templates/typescript/yutori/tools/system.ts index 796bc362..e53ac6fb 100644 --- a/pkg/templates/typescript/yutori/tools/system.ts +++ b/pkg/templates/typescript/yutori/tools/system.ts @@ -95,7 +95,7 @@ export class SystemTools { const result = await this.kernel.browsers.process.exec(this.sessionId, { command: 'bash', args: ['-lc', script], - ...(this.cwd ? { cwd: this.cwd } : {}), + cwd: this.cwd ?? null, timeout_sec: timeoutSec, }); @@ -167,9 +167,11 @@ export class SystemTools { ); } + // The replacer function form is required — a string replacement would + // expand `$$`, `$&`, and friends inside n2's new_string. const updated = args.replace_all ? original.split(oldString).join(newString) - : original.replace(oldString, newString); + : original.replace(oldString, () => newString); await this.kernel.browsers.fs.writeFile(this.sessionId, updated, { path: filePath }); @@ -183,7 +185,7 @@ export class SystemTools { const result = await this.kernel.browsers.process.exec(this.sessionId, { command: 'bash', args: ['-lc', script], - ...(this.cwd ? { cwd: this.cwd } : {}), + cwd: this.cwd ?? null, }); const pid = decode(result.stdout_b64).trim();