From b348891d2e5d5a1b9fe1399998ca6f0bbf59ccf7 Mon Sep 17 00:00:00 2001 From: adamlu123 Date: Wed, 27 May 2026 17:11:53 -0700 Subject: [PATCH] fix(local_browser): drop final_script.py workflow and bound ARIA history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit local_browser.yaml's prompts were copy-pasted from base.yaml's bash_command-driven workspace flow, which doesn't fit a live persistent Playwright session. The agent was told to launch its own browser inside final_script.py while also being told never to import or launch Playwright; and it was told to shell out via `run_command(...)`, a helper that does not exist anywhere in the codebase. Reframe the live-browser mode around its actual capabilities: - Drop final_script.py, plan.md, self_reflect_config.json, final_runs/, image_qa, and self_reflection from the prompts. There is no workspace directory in this mode; the agent drives the live `page` and reports its answer in `final_response` when done. - Override `agent.require_self_reflection_success: false` so the completion gate (which checks for `final_runs/run_/self_reflect_result.json`) does not block `done: true` in this mode. - Override `agent.summary_user_prompt` to talk about live browser state (URL, controls, selectors) instead of workspace artifacts (`plan.md`, `final_script.py`, `final_runs/`) that don't exist. - Document in the yaml header what each step's observation contains (URL / title / stdout / ARIA snapshot / screenshot path) and note that the screenshot is NOT visually attached by default — flipping `model.attach_observation_screenshot: true` enables multimodal image input. Add a context-bounding knob in `AgentConfig`: - `keep_last_n_observations` (default -1, disabled) strips the ARIA snapshot payload from observation messages older than the most recent N, both in the rendered text content and the `extra` observation dict. URL / title / printed stdout are preserved so the agent still knows what it did. local_browser.yaml opts in with N=1. Validated end-to-end against "Search for flights from SEA to JFK on 2026-08-15 to 2026-08-20" via base.yaml + local_browser.yaml + model_openai.yaml: the agent reached the Google Flights results page (round trip, SEA -> JFK, Aug 15 -> Aug 20) in 7 steps with N=1 pruning (vs 9 steps without), and the saved trajectory shows ARIA stripped from the first 6 observations and preserved on the 7th. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/webwright/agents/default.py | 33 +++ src/webwright/config/local_browser.yaml | 315 +++++++----------------- 2 files changed, 127 insertions(+), 221 deletions(-) diff --git a/src/webwright/agents/default.py b/src/webwright/agents/default.py index 02e542e..d05ea86 100644 --- a/src/webwright/agents/default.py +++ b/src/webwright/agents/default.py @@ -39,6 +39,10 @@ class AgentConfig(BaseModel): require_self_reflection_success: bool = False summary_every_n_steps: int = 0 summary_user_prompt: str = DEFAULT_SUMMARY_USER_PROMPT + # Strip the ARIA snapshot payload from observation messages older than the last N + # to bound context growth in browser-driven modes. Any value <= 0 disables pruning + # (default). Opt in per config (e.g. local_browser.yaml sets this to 1). + keep_last_n_observations: int = -1 output_path: Path | None = None @@ -265,8 +269,37 @@ def _tool_gate_error(self) -> str | None: def add_messages(self, *messages: dict[str, Any]) -> list[dict[str, Any]]: self.messages.extend(messages) + self._prune_old_observation_aria_snapshots() return list(messages) + def _prune_old_observation_aria_snapshots(self) -> None: + n = self.config.keep_last_n_observations + if n <= 0: + return + obs_indices = [ + i for i, m in enumerate(self.messages) + if m.get("extra", {}).get("observation") + ] + if len(obs_indices) <= n: + return + placeholder = "(ARIA snapshot pruned; see most recent observation)" + for idx in obs_indices[:-n]: + msg = self.messages[idx] + obs = msg["extra"]["observation"] + aria = obs.get("aria_snapshot", "") + if not aria: + continue + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") in ("text", "input_text"): + text = part.get("text", "") + if aria in text: + part["text"] = text.replace(aria, placeholder) + elif isinstance(content, str) and aria in content: + msg["content"] = content.replace(aria, placeholder) + obs["aria_snapshot"] = "" + def _compact_history(self) -> None: """Summarize the running transcript via an LLM call and reset messages to [system, summary]. diff --git a/src/webwright/config/local_browser.yaml b/src/webwright/config/local_browser.yaml index a1b369d..207c0a2 100644 --- a/src/webwright/config/local_browser.yaml +++ b/src/webwright/config/local_browser.yaml @@ -6,12 +6,31 @@ # -t "Open example.com and report the title" \ # --start-url https://example.com # -# By default this uses real Edge/Chrome over CDP on http://127.0.0.1:9222 -# with ~/.cache/webwright/edge-profile, matching the browser startup -# convention used by the local browser skills. Override with LOCAL_BROWSER_CDP_URL -# / BROWSER_CDP_URL or LOCAL_BROWSER_USER_DATA_DIR if needed. -# local_cdp uses the real browser window size instead of forcing a Playwright -# viewport, so pages lay out like a normal user-controlled browser window. +# This mode runs a live Playwright session: the agent drives `page`, `context`, +# `browser`, and `playwright` directly each turn. There is NO workspace +# directory, NO standalone final script, NO image_qa, and NO self_reflection +# — the agent observes the live page and reports the answer in `final_response` +# when it is done. +# +# What the agent sees each step (via the observation_template below): +# - status / URL / title / printed stdout from the `python_code` step +# - browser console output captured since the previous step +# - ARIA snapshot of the page body (text, every step) — this is the agent's +# primary view of page structure +# - Screenshot PATH as text (the env saves step_.png to disk every step) +# +# The screenshot file is NOT visually attached to the prompt by default. +# `base.yaml` sets `model.attach_observation_screenshot: false`, so the agent +# relies on the ARIA snapshot + printed text. To send the PNG as a real image +# input each step (extra image tokens, slower, costlier), override in this file: +# +# model: +# attach_observation_screenshot: true +# +# Defaults to real Edge/Chrome over CDP on http://127.0.0.1:9222 with +# ~/.cache/webwright/edge-profile. Override with LOCAL_BROWSER_CDP_URL / +# BROWSER_CDP_URL or LOCAL_BROWSER_USER_DATA_DIR if needed. local_cdp uses +# the real browser window size instead of forcing a Playwright viewport. model: action_field: python_code @@ -55,222 +74,120 @@ environment: local_cdp_new_page: true agent: + # No artifact-based verification in this mode; the model decides when it's done. + require_self_reflection_success: false + + # Keep only the most recent observation's ARIA snapshot in context; strip ARIA + # from older observation messages (URL, title, printed stdout are preserved). + # ARIA snapshots are typically ~10-20k chars each and dominate token usage in + # browser-driven loops. The agent still sees what code it ran and what each + # step's URL/title/output were, so navigation works fine with N=1. + keep_last_n_observations: 1 + + # Compaction fires every `summary_every_n_steps` (20, inherited from base.yaml). + # base.yaml's default summary prompt references workspace artifacts (plan.md, + # final_script.py, final_runs/, self_reflection) and `bash_command` that do not + # apply to live-browser mode, so we override it with a prompt tailored to + # browser state. + summary_user_prompt: | + You are about to have your working context compacted to save tokens. + + Write a concise but COMPLETE summary of everything relevant from the conversation above so that a + fresh agent with only this summary (plus the original system prompt and task instructions) can + continue driving the live browser without losing progress. Include: + + - The original task goal and every explicit constraint or filter. + - The current page state: URL, title, key visible labels, which controls/drawers are open, any + filter or selection chips currently applied. + - What has been done so far and what remains, including any blockers or dead ends encountered. + - Key findings worth remembering: stable selectors that worked, ARIA labels, URL patterns, + pitfalls to avoid, and any datum already extracted from the page. + - The next concrete browser action to take. + + Write the summary as plain prose and bullet lists. Do NOT issue a new `python_code`. Do NOT set + `done=true`. Put the entire summary in the `thought` field and leave `python_code` and + `final_response` empty. + system_template: | - You are a web agent operating through a local terminal + workspace harness. + You are a web agent driving a live local browser session. Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: { "thought": "", "python_code": "", "done": false, - "final_response": "" + "final_response": "" } Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. - Global constraints: - - Put exactly one async Python browser/workspace step in the `python_code` string. Never emit raw Python outside that field. Use the provided async variables `page`, `context`, `browser`, `playwright`, `asyncio`, and `task`; use `run_command("...")` for shell commands whose output you need in the next observation. + ## Global Constraints + - Put exactly one async Python step in `python_code`. The harness already exposes `page`, `context`, `browser`, `playwright`, `asyncio`, and `task` — drive the live browser through these. Never import, launch, or close Playwright yourself; never close `page`, `context`, `browser`, or `playwright`. + - The live browser state is persistent across steps. Reuse it; do not re-navigate from scratch every turn unless something has gone wrong. + - There is NO workspace directory in this mode. Do NOT write artifact files. Drive the browser, observe the page, and report. - Escape newlines and quotes properly so the whole object remains valid JSON. - - You should reason internally, then execute one Python step, then inspect the next observation. - - The live Playwright browser state is persistent across steps. Do not import, launch, or close Playwright yourself; do not close `page`, `context`, `browser`, or `playwright`. - - Step screenshots are NOT automatically attached to your prompt in this benchmark variant. If you need visual interpretation, you must invoke the image QA tool yourself. - - Set `"done": true` only when the task goal is complete and `final_script.py` is the final artifact. - - NEVER set `"done": true` in the same response as a non-empty `python_code`. Declare done in a SEPARATE response AFTER you have already executed and verified the final state in a prior step. - - In `thought`, write in detail your observation, reasoning, and next step. - - Do NOT install additional packages with pip, apt, or any other package manager. All required packages (playwright, httpx, etc.) are already installed. + - Reason internally, then execute one Python step, then inspect the next observation. + - Set `"done": true` ONLY when you can fully answer the task. Put the user-visible answer in `final_response`. + - NEVER set `"done": true` in the same response as a non-empty `python_code`. Declare done in a SEPARATE response AFTER the prior observation confirmed the answer. + - Do NOT install additional packages with pip, apt, or any other package manager. All required packages are already installed. ## Browser Mode - The harness exposes a live browser session to your Python steps. - - When `browser_mode=local_cdp`: use the already-connected Chrome/Edge page over CDP. - - When `browser_mode=local_persistent` or `local_launch`: use the already-created Playwright page. + The harness has already started the browser for you: + - `browser_mode=local_cdp` (default): an already-connected Chrome/Edge page over CDP. + - `browser_mode=local_persistent` or `local_launch`: an already-created Playwright page. - ## Playwright Examples - Example response (rendered for readability - in practice you emit a single JSON object on one logical message): + ## Playwright Example + + Example response (rendered for readability — in practice you emit a single JSON object on one logical message): ``` { - "thought": "Run one Playwright step against the live page, capture screenshots, and print aria evidence for the next step.", - "python_code": "from pathlib import Path\nWORKSPACE = Path(\"{{ workspace_dir }}\")\nSCREENSHOTS = WORKSPACE / \"screenshots\"\nSCREENSHOTS.mkdir(parents=True, exist_ok=True)\n\nawait page.goto(\"{{ start_url }}\", wait_until=\"domcontentloaded\")\nawait page.screenshot(path=str(SCREENSHOTS / \"final_execution_1_open_start_page.png\"), full_page=False)\n\nprint(\"URL:\", page.url)\nprint(\"TITLE:\", await page.title())\nprint(\"ARIA:\", await page.locator(\"body\").aria_snapshot())", + "thought": "Navigate to the start URL and print the URL, title, and a short ARIA snapshot for the next step.", + "python_code": "await page.goto(\"{{ start_url }}\", wait_until=\"domcontentloaded\")\nprint(\"URL:\", page.url)\nprint(\"TITLE:\", await page.title())\nprint(\"ARIA:\", await page.locator(\"body\").aria_snapshot())", "done": false, "final_response": "" } ``` (The `python_code` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) - ## Helpful Command Patterns - - - Inspect a script: - ``` - run_command("sed -n '1,220p' final_script.py") - ``` - - Prefer incremental edits once the file exists, keeping patch, execution, and verification in one `python_code`. - - Inspect the latest run artifacts: - ``` - run_command("ls -R final_runs && sed -n '1,200p' final_runs/run_003/final_script_log.txt") - ``` - - Ask a grounded question about a saved screenshot: - ``` - run_command('python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/explore.png --question "Is the BMW filter chip visibly selected?"') - ``` - - Final multi-image verification with action log: - ``` - run_command('RUN_DIR="final_runs/run_003" && ACTION_LOG="$(tail -n 80 "${RUN_DIR}/final_script_log.txt")" && python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image "${RUN_DIR}/screenshots/final_execution_1_apply_constraint.png" --image "${RUN_DIR}/screenshots/final_execution_2_sort.png" --image "${RUN_DIR}/screenshots/final_execution_3_final_state.png" --question "Final script critical-point action log:\n${ACTION_LOG}\n\nUsing the action log and all screenshots together, are all required constraints visibly satisfied and are results displayed?"') - ``` - ## Rules - - **Always Avoid taking full page screenshot using Playwright, use viewport 1280x1800 ** (exploration, debugging, and final-run screenshots alike). Never do `page.screenshot(full_page=True)`. - - After a file already exists, prefer incremental edits over rewriting the whole file. - - Use stable selectors and current-run evidence. + - Use stable selectors and current-observation evidence; do not guess UI interactions. + - Hidden controls (drawers, accordions, dropdowns, mobile filter panels) must be opened before deciding a control is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. - - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in ``. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in `final_response`. - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. - - Make sure to save as more critical points as possible, especially those that show the application of required filters or constraints, and the final result display. The more evidence you save, the higher chance the judge will verify the successful completion of the task. - It also needs to save the final response of the task in `final_runs/run_/final_script_log.txt` in the end. - - ## Task Reflection Tool - - Do NOT hand-roll a `judge.py` that loops `image_qa`. Use the built-in - `webwright.tools.self_reflection` CLI. - - 1. Stage 1 - score each screenshot against the full set of critical points with a - single (system, user+image) prompt pair. The tool parses a `Score: 1-5` and - `Reasoning: ` from each response and retries on parse failure. - 2. Stage 2 - drop every per-image `Reasoning` into the final user prompt template - via `{image_reasonings}`, inject the latest run's `final_script_log.txt` via - `{action_history_log}`, attach EVERY screenshot (no filtering), and make ONE - aggregated call that must end with `Status: success` or `Status: failure`. - - Your job is to AUTHOR the four prompts ONCE and reuse them for every - self_reflection invocation in this run. The tool handles parallel per-image - scoring, final aggregation, and verdict parsing. - - **CLI interface:** - ``` - python -m webwright.tools.self_reflection \ - --config {{ workspace_dir }}/self_reflect_config.json \ - --workspace-dir "{{ workspace_dir }}" \ - --output {{ workspace_dir }}/final_runs/run_/self_reflect_result.json - ``` - - Exit code is 0 on PASS, 1 on FAIL / unparsed. The `--output` file is a JSON document - containing per-image records (`Score`, `Reasoning`, `Response`), the image path - list, the full final-stage prompt, the model's final response, and - `predicted_label` (1=success, 0=failure, null=unparsed). You MUST run self_reflection before - declaring done. - - **self_reflect_config.json schema (authored once, prompts only):** - ```json - { - "image_judge_system_prompt": "...see below...", - "image_judge_user_prompt": "...see below...", - "final_verdict_system_prompt": "...see below...", - "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." - } - ``` - Any `_file` variant (e.g. `image_judge_user_prompt_file`) may point to a - text file on disk instead of inlining the prompt - useful when prompts contain - many curly braces or embedded JSON. - - **Required prompt content (you MUST include all of this in the JSON you write):** - - - `image_judge_system_prompt`: instruct the model to act as a harsh evaluator and - return ONLY two labelled lines: - ``` - Reasoning: <1-2 sentences describing what the screenshot shows and which critical - points it provides evidence for or against> - Score: - ``` - Do NOT ask for JSON. The tool parses the labelled lines directly. - - - `image_judge_user_prompt`: embed the task description and the full numbered - critical-point list from `plan.md`. Tell the model to consider ALL critical - points when scoring this single image and to be harsh when evidence is - ambiguous or partially occluded. - - - `final_verdict_system_prompt`: instruct the model to be a harsh aggregated judge - and to end its reply with EXACTLY `Status: success` or `Status: failure` on its - own line. Require a `Thoughts:` block before that line that evaluates every - critical point. The tool extracts the verdict from the trailing `Status:` line. - - - `final_verdict_user_prompt`: embed the task description and the numbered - critical-point list, and include the literal tokens `{action_history_log}` and - `{image_reasonings}` where you want the final run's `final_script_log.txt` - content and the per-image reasonings injected. Do NOT hard-code a specific - run's `final_script_log.txt`. The tool renders those tokens with Python - `str.format`, so any other literal curly braces in this string MUST be doubled - (write {% raw %}`{{`{% endraw %} and {% raw %}`}}`{% endraw %} in the JSON to emit a literal `{` or `}`). - - **Verdict extraction:** the tool parses `Status: success|failure` from the last - line of the final-stage response. Missing or malformed `Status:` counts as FAIL - (exit code 1). Keep the verdict line clean. - - **Robustness:** per-image parse failures are retried up to 3 times and then - recorded with `Score: 0, ParseFailed: true` without failing the whole run. - Transient model-API HTTP errors are retried with exponential backoff. - - ## Completion Gate - - Set `"done": true` ONLY if ALL of the following are true: - 1. `plan.md` exists and every critical point is enumerated as a checklist item. - 2. `self_reflect_config.json` exists at the workspace root with all four prompts populated - for `self_reflection`. - 3. `final_script.py` was executed successfully from scratch inside a - `final_runs/run_/` folder, producing `final_script_log.txt` and all - critical-point screenshots. - 4. `python -m webwright.tools.self_reflection --config self_reflect_config.json - --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json` - was executed against that run, exited 0, and wrote - `final_runs/run_/self_reflect_result.json` with `"predicted_label": 1`. - 5. You have run `ls -R final_runs/run_`, - `ls -R final_runs/run_/screenshots`, and - `cat final_runs/run_/final_script_log.txt` to confirm the artifacts and - logs are in place. - - Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is - not 1, if the run folder is missing, if required screenshots are missing, if the - script failed to run, or if the checklist in `plan.md` is incomplete. If - `self_reflection` fails, diagnose the specific issue (wrong filter value, missing - control, missing confirmation, missing screenshot, etc.), fix `final_script.py`, - re-run it in a new `final_runs/run_/` folder, and re-run `self_reflection` - against the new run. Do NOT edit `self_reflect_config.json` between attempts unless a - prompt itself is objectively wrong. instance_template: | Task: {{ task }} {% if task_id %}Task ID: {{ task_id }} {% endif %}{% if start_url %}Start URL: {{ start_url }} - {% endif %}Workspace root: {{ workspace_dir }} - Task metadata JSON: {{ task_metadata_path }} - Required final script path: {{ final_script_path }} + {% endif %}Browser mode: {{ browser_mode }} # Task Instructions - You're solving a user-specified web task through a stateless local terminal + workspace harness. + You're solving a user-specified web task by driving a live local browser session. - This is an interactive process where you reason, execute exactly one async Python browser/workspace step, inspect the result, and then produce your next step. You have a single session — context is preserved across all steps, so there is no need to reload state between turns. + This is an interactive process where you reason, execute exactly one async Python browser step, inspect the result, and then produce your next step. The browser state and your context are both persistent across steps — no need to reload state between turns. ## Harness Rules - - Work only inside `{{ workspace_dir }}`. - - Keep generated code, screenshots, logs, scratch files, and notes **only** in `{{ workspace_dir }}`. - - The required final artifact is `{{ final_script_path }}`. - - Create `final_runs/run_/` folders for every clean execution of the final script. Use an integer ID higher than any that already exists for each new attempt. - - Store each run's `final_script.py`, `final_script_log.txt`, and final verification screenshots **only** inside that run folder. - - The browser mode is `{{ browser_mode }}`. Match your generated scripts to that mode. + - There is NO workspace directory. Do NOT write any files; do NOT create `final_runs/`, `screenshots/`, `plan.md`, `final_script.py`, or any log. + - Drive the live `page`, `context`, `browser`, and `playwright` variables. Do NOT re-launch Playwright. + - Do NOT invoke `webwright.tools.image_qa` or `webwright.tools.self_reflection`. + - Browser mode is `{{ browser_mode }}`. ## Web Task Rules - - Do not guess UI interactions. Use printed evidence from the current run. + - Do not guess UI interactions. Use printed evidence from the current observation. - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. - - Save final verification screenshots inside the active `final_runs/run_/screenshots/` folder. - Print concise ARIA snapshots, URLs, titles, visible labels, and any extracted state needed for the next step. ## Task Success Criteria @@ -278,59 +195,15 @@ agent: 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. 3. Requirements must be applied through filters, not embedded in a broad search query. - 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement - no broadening or narrowing. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement — no broadening or narrowing. 5. Tasks requiring a submission action or results display need that action to be taken. 6. Empty results are OK if the correct action was performed. 7. All explicit filters must use site controls when those controls exist. 8. If a site control does not exist, verify the constraint directly from page content. - ## Image QA Tool - - - Use image_qa during exploration to inspect screenshots and verify UI state: - `python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/example.png --question "inspect prompt"` - - Use multiple `--image` flags for combined visual verification. - - image_qa returns JSON with `answer`, `evidence`, `unknown`, and `confidence` fields. - - ## Recommended Workflow - - 1. **Planning**: Parse the task into a list of critical points — every explicit constraint, filter, sort, selection, or datum that must be satisfied. Write them to `plan.md` as a checklist: - ``` - # Critical Points - - [ ] CP1: - - [ ] CP2: - ... - ``` - Each critical point must be independently verifiable from a screenshot or log entry. - - 2. **Author self_reflect_config.json (once)**: Write `{{ workspace_dir }}/self_reflect_config.json` containing only the four prompts (`image_judge_system_prompt`, `image_judge_user_prompt`, `final_verdict_system_prompt`, `final_verdict_user_prompt`) for `webwright.tools.self_reflection`. Embed the full critical-point list from `plan.md` and the task description into the user prompts, but keep the prompts generic — this file is reused verbatim for every `self_reflection` invocation, so do NOT hard-code a specific run id, screenshot filename, or `final_script_log.txt` content. - - 3. **Exploration**: Inspect `task.json`, create exploration scripts, identify every required filter control. Use `image_qa` during exploration to verify UI state. - - 4. **Final script**: Write `final_script.py`, run it once in a new `final_runs/run_/` folder. The script must produce screenshots and action logs as described in **Final Script Instrumentation**. - - 5. **Run self_reflection**: Execute `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json`. The tool auto-attaches every screenshot in the latest `final_runs/run_*/screenshots/` folder (default `--auto-latest-run final_runs`) — you do NOT pass an image list. If the tool exits non-zero or `predicted_label != 1`, diagnose the specific issue, fix `final_script.py`, re-run it in a new `final_runs/run_/` folder, and re-invoke `self_reflection` against the new run. Do NOT edit `self_reflect_config.json` between attempts. - - 6. **Declare done**: Set `"done": true` ONLY after `self_reflection` exits 0 and `self_reflect_result.json` reports `"predicted_label": 1` for the latest run. The external judge reads that same `self_reflect_result.json` as the final verdict. Declaring done in any other state is a failure. - - ## Final Script Instrumentation - - `final_script.py` must: - - be stored as `final_runs/run_/final_script.py` - - save critical-point screenshots as `final_runs/run_/screenshots/final_execution__.png` - - create or reset `final_runs/run_/final_script_log.txt` at the start of each clean run - - write `step action: ` to the log for every constraint-relevant interaction - - each screenshot should correspond to a critical point from `plan.md` so that `self_reflection` can verify it - - This instrumentation is mandatory because both `self_reflection` and the external judge evaluate those screenshots and action logs. - - ## Completion Gate - - Set `"done": true` ONLY if ALL of the following are true: - 1. `plan.md` exists with all critical points identified. - 2. `self_reflect_config.json` exists with all four prompts populated for `self_reflection`. - 3. `final_script.py` was run from scratch in a `final_runs/run_/` folder. - 4. `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json` was executed against that run, exited 0, and wrote `final_runs/run_/self_reflect_result.json` with `"predicted_label": 1`. - 5. `ls -R final_runs/run_` and `cat final_runs/run_/final_script_log.txt` confirm the expected artifacts. + ## Workflow - Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is not 1, if the run folder is missing, if required screenshots are missing, or if `self_reflection` has not been run against the latest `final_runs/run_/`. + 1. Reason about the task and identify the constraints. + 2. Drive the live `page` one step at a time. Print evidence (URL, title, ARIA snippets, extracted text) so the next step has context. + 3. When the task is fully satisfied and you can state the answer, set `"done": true` in a SEPARATE turn and put the user-visible answer in `final_response`. Do NOT pair `done: true` with a non-empty `python_code`.