From ba029c1dea601b01a94685e3eda62400bde9ff93 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Tue, 23 Dec 2025 14:20:51 -0800 Subject: [PATCH] chore: add expectAria to perform --- docs/src/api/class-page.md | 89 +--- docs/src/api/class-pageagent.md | 82 ++++ docs/src/api/params.md | 27 +- docs/src/test-api/class-fullconfig.md | 2 +- docs/src/test-api/class-testconfig.md | 2 +- .../add-multiple-todos-2.spec.ts | 23 - .../add-multiple-todos-2.spec.ts-cache.json | 112 ----- .../tests/perform/adding-new-todos.spec.ts | 54 +++ .../adding-new-todos.spec.ts-cache.json | 258 +++++++++++ .../tests/perform/completing-todos.spec.ts | 75 +++ .../completing-todos.spec.ts-cache.json | 429 ++++++++++++++++++ .../tests/perform/editing-todos.spec.ts | 63 +++ .../perform/editing-todos.spec.ts-cache.json | 249 ++++++++++ package-lock.json | 8 +- package.json | 3 +- packages/injected/src/ariaSnapshot.ts | 2 +- packages/playwright-client/types/types.d.ts | 154 ++++--- .../playwright-core/ThirdPartyNotices.txt | 6 +- .../bundles/mcp/package-lock.json | 8 +- .../playwright-core/bundles/mcp/package.json | 2 +- packages/playwright-core/src/client/api.ts | 1 + packages/playwright-core/src/client/page.ts | 14 +- .../playwright-core/src/client/pageAgent.ts | 42 ++ .../playwright-core/src/protocol/validator.ts | 17 +- .../src/server/agent/actionRunner.ts | 13 +- .../src/server/agent/actions.ts | 19 +- .../src/server/agent/codegen.ts | 5 +- .../src/server/agent/context.ts | 2 +- .../src/server/agent/expectTools.ts | 113 +++++ .../server/agent/{agent.ts => pageAgent.ts} | 81 +++- .../agent/{tools.ts => performTools.ts} | 96 +--- .../src/server/agent/{backend.ts => tool.ts} | 32 +- .../src/server/dispatchers/pageDispatcher.ts | 14 +- packages/playwright-core/src/utils.ts | 1 + .../src/utils/isomorphic/protocolMetainfo.ts | 5 +- .../src/utils/isomorphic/yaml.ts | 94 ++++ packages/playwright-core/types/types.d.ts | 154 ++++--- packages/playwright/src/agents/agent.ts | 98 ---- packages/playwright/src/agents/agentParser.ts | 12 +- .../playwright/src/agents/generateAgents.ts | 2 +- packages/playwright/types/test.d.ts | 6 +- packages/protocol/src/channels.d.ts | 33 +- packages/protocol/src/protocol.yml | 14 +- tests/library/perform-task.spec.ts | 10 +- tests/mcp/browser.eval.ts | 46 -- tests/mcp/fixtures.ts | 30 -- tests/mcp/playwright.config.ts | 9 - utils/generate_types/overrides.d.ts | 2 + 48 files changed, 1911 insertions(+), 702 deletions(-) create mode 100644 docs/src/api/class-pageagent.md delete mode 100644 examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts delete mode 100644 examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json create mode 100644 examples/todomvc/tests/perform/adding-new-todos.spec.ts create mode 100644 examples/todomvc/tests/perform/adding-new-todos.spec.ts-cache.json create mode 100644 examples/todomvc/tests/perform/completing-todos.spec.ts create mode 100644 examples/todomvc/tests/perform/completing-todos.spec.ts-cache.json create mode 100644 examples/todomvc/tests/perform/editing-todos.spec.ts create mode 100644 examples/todomvc/tests/perform/editing-todos.spec.ts-cache.json create mode 100644 packages/playwright-core/src/client/pageAgent.ts create mode 100644 packages/playwright-core/src/server/agent/expectTools.ts rename packages/playwright-core/src/server/agent/{agent.ts => pageAgent.ts} (68%) rename packages/playwright-core/src/server/agent/{tools.ts => performTools.ts} (70%) rename packages/playwright-core/src/server/agent/{backend.ts => tool.ts} (62%) create mode 100644 packages/playwright-core/src/utils/isomorphic/yaml.ts delete mode 100644 packages/playwright/src/agents/agent.ts delete mode 100644 tests/mcp/browser.eval.ts diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 19aa0d94f5d82..23e8aad37c892 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -720,6 +720,10 @@ current working directory. Raw CSS content to be injected into frame. +## property: Page.agent +* since: v1.58 +- type: <[PageAgent]> + ## async method: Page.bringToFront * since: v1.8 @@ -2035,49 +2039,6 @@ Name of the function on the window object Callback function which will be called in Playwright's context. -## async method: Page.extract -* since: v1.58 -- returns: <[Object]> - - `result` <[any]> - - `usage` <[Object]> - - `turns` <[int]> - - `inputTokens` <[int]> - - `outputTokens` <[int]> - -Extract information from the page using the agentic loop, return it in a given Zod format. - -**Usage** - -```js -await page.extract('List of items in the cart', z.object({ - title: z.string().describe('Item title to extract'), - price: z.string().describe('Item price to extract'), -}).array()); -``` - -### param: Page.extract.query -* since: v1.58 -- `query` <[string]> - -Task to perform using agentic loop. - -### param: Page.extract.schema -* since: v1.58 -- `schema` <[z.ZodSchema]> - -### option: Page.extract.maxTokens -* since: v1.58 -- `maxTokens` <[int]> - -Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. -Defaults to context-wide value specified in `agent` property. - -### option: Page.extract.maxTurns -* since: v1.58 -- `maxTurns` <[int]> - -Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. - ## async method: Page.fill * since: v1.8 * discouraged: Use locator-based [`method: Locator.fill`] instead. Read more about [locators](../locators.md). @@ -3051,48 +3012,6 @@ Whether or not to generate tagged (accessible) PDF. Defaults to `false`. Whether or not to embed the document outline into the PDF. Defaults to `false`. -## async method: Page.perform -* since: v1.58 -- returns: <[Object]> - - `usage` <[Object]> - - `turns` <[int]> - - `inputTokens` <[int]> - - `outputTokens` <[int]> - -Perform action using agentic loop. - -**Usage** - -```js -await page.perform('Click submit button'); -``` - -### param: Page.perform.task -* since: v1.58 -- `task` <[string]> - -Task to perform using agentic loop. - -### option: Page.perform.key -* since: v1.58 -- `key` <[string]> - -All the agentic actions are converted to the Playwright calls and are cached. -By default, they are cached globally with the `task` as a key. This option allows controlling the cache key explicitly. - -### option: Page.perform.maxTokens -* since: v1.58 -- `maxTokens` <[int]> - -Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. -Defaults to context-wide value specified in `agent` property. - -### option: Page.perform.maxTurns -* since: v1.58 -- `maxTurns` <[int]> - -Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. - ## async method: Page.press * since: v1.8 diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md new file mode 100644 index 0000000000000..d3f2f4ee917ce --- /dev/null +++ b/docs/src/api/class-pageagent.md @@ -0,0 +1,82 @@ +# class: PageAgent +* since: v1.58 + + +## async method: PageAgent.expect +* since: v1.58 + +Expect certain condition to be met. + +**Usage** + +```js +await page.agent.expect('"0 items" to be reported'); +``` + +### param: PageAgent.expect.expectation +* since: v1.58 +- `expectation` <[string]> + +Expectation to assert. + +### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%% +* since: v1.58 + +## async method: PageAgent.extract +* since: v1.58 +- returns: <[Object]> + - `result` <[any]> + - `usage` <[Object]> + - `turns` <[int]> + - `inputTokens` <[int]> + - `outputTokens` <[int]> + +Extract information from the page using the agentic loop, return it in a given Zod format. + +**Usage** + +```js +await page.agent.extract('List of items in the cart', z.object({ + title: z.string().describe('Item title to extract'), + price: z.string().describe('Item price to extract'), +}).array()); +``` + +### param: PageAgent.extract.query +* since: v1.58 +- `query` <[string]> + +Task to perform using agentic loop. + +### param: PageAgent.extract.schema +* since: v1.58 +- `schema` <[z.ZodSchema]> + +### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%% +* since: v1.58 + + +## async method: PageAgent.perform +* since: v1.58 +- returns: <[Object]> + - `usage` <[Object]> + - `turns` <[int]> + - `inputTokens` <[int]> + - `outputTokens` <[int]> + +Perform action using agentic loop. + +**Usage** + +```js +await page.agent.perform('Click submit button'); +``` + +### param: PageAgent.perform.task +* since: v1.58 +- `task` <[string]> + +Task to perform using agentic loop. + +### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%% +* since: v1.58 diff --git a/docs/src/api/params.md b/docs/src/api/params.md index a4f6a5f53582d..21c9a708ead5f 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -380,7 +380,32 @@ Emulates consistent window screen size available inside web page via `window.scr - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10. - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited. -Agent settings for [`method: Page.perform`] and [`method: Page.extract`]. +Agent settings for [`property: Page.agent`]. + +## page-agent-key +* since: v1.58 +- `key` <[string]> + +All the agentic actions are converted to the Playwright calls and are cached. +By default, they are cached globally with the `task` as a key. This option allows controlling the cache key explicitly. + +## page-agent-max-tokens +* since: v1.58 +- `maxTokens` <[int]> + +Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. +Defaults to context-wide value specified in `agent` property. + +## page-agent-max-turns +* since: v1.58 +- `maxTurns` <[int]> + +Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + +## page-agent-call-options-v1.58 +- %%-page-agent-key-%% +- %%-page-agent-max-tokens-%% +- %%-page-agent-max-turns-%% ## fetch-param-url - `url` <[string]> diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 2a97b248a8325..634c48ba88c15 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -108,7 +108,7 @@ Base directory for all relative paths used in the reporters. * since: v1.58 - type: <['RunAgentsMode]<"all"|"missing"|"none">> -Whether to run LLM agent for [`method: Page.perform`]: +Whether to run LLM agent for [`property: Page.agent`]: * "all" disregards existing cache and performs all actions via LLM * "missing" only performs actions that don't have generated cache actions * "none" does not talk to LLM at all, relies on the cached actions (default) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index d171115cd5c65..5a9251062ecdc 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -518,7 +518,7 @@ export default defineConfig({ * since: v1.58 - type: ?<['RunAgentsMode]<"all"|"missing"|"none">> -Whether to run LLM agent for [`method: Page.perform`]: +Whether to run LLM agent for [`property: Page.agent`]: * "all" disregards existing cache and performs all actions via LLM * "missing" only performs actions that don't have generated cache actions * "none" does not talk to LLM at all, relies on the cached actions (default) diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts deleted file mode 100644 index a0bdcc180f811..0000000000000 --- a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { test } from '../fixtures'; - -test.use({ - agent: { - provider: 'github', - model: 'gpt-4.1', - } -}); - -test.describe('Adding New Todos', () => { - test('should add multiple todos', async ({ page }) => { - page.on('agentturn', turn => { - console.log('agentturn', turn.role, turn.message); - }) - await page.perform(`Add "Buy groceries", todo item`); - await page.perform(`Add "Walk the dog" todo item`); - await page.perform(`Add "Read a book" todo item`); - - await page.perform(`Ensure each todo has an unchecked checkbox`); - await page.perform(`Ensure counter shows "3 items left" (plural)`); - await page.perform(`Ensure input field is cleared`); - }); -}); diff --git a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json b/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json deleted file mode 100644 index 620c26dfdec9d..0000000000000 --- a/examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "Add \"Buy groceries\", todo item": { - "timestamp": 1765848997203, - "actions": [ - { - "method": "fill", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "text": "Buy groceries", - "submit": true, - "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');", - "intent": "Add the new todo item 'Buy groceries' by typing into the input field and submitting." - }, - { - "method": "expectVisible", - "selector": "internal:text=\"Buy groceries\"i", - "code": "await expect(page.getByText('Buy groceries')).toBeVisible();", - "intent": "Verify that the todo item 'Buy groceries' appears in the list, confirming addition." - } - ] - }, - "Add \"Read a book\" todo item": { - "timestamp": 1765849025987, - "actions": [ - { - "method": "fill", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "text": "Read a book", - "submit": true, - "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');\nawait page.keyboard.press('Enter');", - "intent": "Add a new todo item called 'Read a book' by typing it into the input and submitting." - }, - { - "method": "expectVisible", - "selector": "internal:text=\"Read a book\"i", - "code": "await expect(page.getByText('Read a book')).toBeVisible();", - "intent": "Verify that the todo item 'Read a book' is now visible on the list to confirm it was added successfully." - } - ] - }, - "Add \"Walk the dog\" todo item": { - "timestamp": 1765849011629, - "actions": [ - { - "method": "fill", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "text": "Walk the dog", - "submit": true, - "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');\nawait page.keyboard.press('Enter');", - "intent": "Add a new todo item 'Walk the dog' by typing into the input field and submitting." - }, - { - "method": "expectVisible", - "selector": "internal:text=\"Walk the dog\"i", - "code": "await expect(page.getByText('Walk the dog')).toBeVisible();", - "intent": "Verify that 'Walk the dog' appears in the list of todo items, confirming the item was added successfully." - } - ] - }, - "Ensure each todo has an unchecked checkbox": { - "timestamp": 1765908604120, - "actions": [ - { - "method": "expectValue", - "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", - "type": "checkbox", - "value": "false", - "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", - "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked." - }, - { - "method": "expectValue", - "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", - "type": "checkbox", - "value": "false", - "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", - "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked." - }, - { - "method": "expectValue", - "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", - "type": "checkbox", - "value": "false", - "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", - "intent": "Verify that the first todo's checkbox is unchecked.\nVerify that the second todo's checkbox is unchecked.\nVerify that the third todo's checkbox is unchecked." - } - ] - }, - "Ensure counter shows \"3 items left\" (plural)": { - "timestamp": 1765908755394, - "actions": [ - { - "method": "expectVisible", - "selector": "internal:text=\"3 items left\"i", - "code": "await expect(page.getByText('3 items left')).toBeVisible();", - "intent": "Verify that the counter displays '3 items left' to ensure proper pluralization for 3 items remaining in the todo list." - } - ] - }, - "Ensure input field is cleared": { - "timestamp": 1765908762823, - "actions": [ - { - "method": "expectValue", - "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", - "type": "textbox", - "value": "", - "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toHaveValue('');", - "intent": "Check that the input field is cleared (empty)." - } - ] - } -} \ No newline at end of file diff --git a/examples/todomvc/tests/perform/adding-new-todos.spec.ts b/examples/todomvc/tests/perform/adding-new-todos.spec.ts new file mode 100644 index 0000000000000..76c169755cf33 --- /dev/null +++ b/examples/todomvc/tests/perform/adding-new-todos.spec.ts @@ -0,0 +1,54 @@ +import { test } from '../fixtures'; + +test.use({ + agent: { + provider: 'github', + model: 'gpt-4.1', + } +}); + +test.describe('Adding New Todos', () => { + + test('should add single valid todo', async ({ page }) => { + await page.agent.perform(`Click in the "What needs to be done?" input field`); + await page.agent.perform(`Add "Buy groceries" todo item`); + + await page.agent.expect(`Todo item "Buy groceries" appears in the list`); + await page.agent.expect(`Counter shows "1 item left"`); + await page.agent.expect(`Input field is cleared and ready for next entry`); + await page.agent.expect(`"Mark all as complete" checkbox becomes visible`); + }); + + test('should add multiple todos', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries", todo item`); + await page.agent.perform(`Add "Walk the dog" todo item`); + await page.agent.perform(`Add "Read a book" todo item`); + + await page.agent.expect('All three todos appear in the list in order of creation'); + await page.agent.expect(`Each todo has an unchecked checkbox`); + await page.agent.expect(`Counter shows "3 items left" (plural)`); + await page.agent.expect(`Input field is cleared`); + }); + + + test('should reject empty todo', async ({ page }) => { + await page.agent.perform(`Click in the "What needs to be done?" input field`); + await page.agent.perform(`Press Enter without typing any text`); + + await page.agent.expect(`No todo is added to the list`); + await page.agent.expect(`Todo list remains empty`); + await page.agent.expect(`Counter is not displayed`); + await page.agent.expect(`Input field remains focused`); + }); + + test('should add todo with special characters', async ({ page }) => { + await page.agent.perform(`Add "Test with special chars: @#$%^&*()" todo item`); + await page.agent.expect(`Todo item "Test with special chars: @#$%^&*()" appears in the list`); + }); + + test('should add todo with long text', async ({ page }) => { + await page.agent.perform(`Add "This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues" todo item`); + await page.agent.expect(`Todo item "This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues" appears in the list`); + }); + +}); diff --git a/examples/todomvc/tests/perform/adding-new-todos.spec.ts-cache.json b/examples/todomvc/tests/perform/adding-new-todos.spec.ts-cache.json new file mode 100644 index 0000000000000..4b6626fbd09a9 --- /dev/null +++ b/examples/todomvc/tests/perform/adding-new-todos.spec.ts-cache.json @@ -0,0 +1,258 @@ +{ + "\"Mark all as complete\" checkbox becomes visible": { + "timestamp": 1766801077373, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "code": "await expect(page.getByRole('checkbox', { name: '❯Mark all as complete' })).toBeVisible();", + "intent": "Verify that the 'Mark all as complete' checkbox is visible to confirm expectation." + } + ] + }, + "Add \"Buy groceries\" todo item": { + "timestamp": 1766801046868, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Buy groceries", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');", + "intent": "Add new todo item called 'Buy groceries' to the list." + } + ] + }, + "Click in the \"What needs to be done?\" input field": { + "timestamp": 1766801039187, + "actions": [ + { + "method": "click", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "options": {}, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).click();", + "intent": "Click inside the 'What needs to be done?' input field to focus it and allow user input." + } + ] + }, + "Counter shows \"1 item left\"": { + "timestamp": 1766801062200, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"1 item left\"i", + "code": "await expect(page.getByText('1 item left')).toBeVisible();", + "intent": "Assert that the counter displays '1 item left' to confirm the correct state." + } + ] + }, + "Input field is cleared and ready for next entry": { + "timestamp": 1766801069802, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "type": "textbox", + "value": "", + "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toHaveValue('');", + "intent": "Verify the input field is cleared and ready for next entry by asserting its value is empty." + } + ] + }, + "Todo item \"Buy groceries\" appears in the list": { + "timestamp": 1766801054693, + "actions": [ + { + "method": "expectAria", + "template": "- list:\n - listitem: Buy groceries", + "code": "await expect(page.locator('body')).toMatchAria(`\n- list:\n - listitem: Buy groceries\n`);", + "intent": "Assert that the todo item 'Buy groceries' appears in the list on the page, ensuring the expected task is present." + } + ] + }, + "Add \"Buy groceries\", todo item": { + "timestamp": 1766801086651, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Buy groceries", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');", + "intent": "Add the todo item 'Buy groceries' by typing it into the new todo textbox and submitting." + } + ] + }, + "Add \"Read a book\" todo item": { + "timestamp": 1766801102073, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Read a book", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');\nawait page.keyboard.press('Enter');", + "intent": "Add a new todo item 'Read a book' by typing in the new todo textbox and submitting." + } + ] + }, + "Add \"Walk the dog\" todo item": { + "timestamp": 1766801094335, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Walk the dog", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');\nawait page.keyboard.press('Enter');", + "intent": "Add a new todo item 'Walk the dog' by typing it into the main input box and submitting." + } + ] + }, + "All three todos appear in the list in order of creation": { + "timestamp": 1766801109675, + "actions": [ + { + "method": "expectAria", + "template": "- list:\n - listitem: Buy groceries\n - listitem: Walk the dog\n - listitem: Read a book", + "code": "await expect(page.locator('body')).toMatchAria(`\n- list:\n - listitem: Buy groceries\n - listitem: Walk the dog\n - listitem: Read a book\n`);", + "intent": "Verify all three todos appear in the list in order of creation: 'Buy groceries', 'Walk the dog', 'Read a book'." + } + ] + }, + "Counter shows \"3 items left\" (plural)": { + "timestamp": 1766801135977, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"3 items left\"i", + "code": "await expect(page.getByText('3 items left')).toBeVisible();", + "intent": "Verify that the counter displays the plural form '3 items left' as expected." + } + ] + }, + "Each todo has an unchecked checkbox": { + "timestamp": 1766801128833, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that the checkbox for the 'Buy groceries' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Walk the dog' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Read a book' todo is unchecked, confirming the todo's active status." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that the checkbox for the 'Buy groceries' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Walk the dog' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Read a book' todo is unchecked, confirming the todo's active status." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that the checkbox for the 'Buy groceries' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Walk the dog' todo is unchecked, confirming the todo's active status.\nAssert that the checkbox for the 'Read a book' todo is unchecked, confirming the todo's active status." + } + ] + }, + "Input field is cleared": { + "timestamp": 1766801143243, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "type": "textbox", + "value": "", + "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toHaveValue('');", + "intent": "Verify that the input field is cleared as expected, i.e., its value should be empty." + } + ] + }, + "Counter is not displayed": { + "timestamp": 1766801156305, + "actions": [] + }, + "Input field remains focused": { + "timestamp": 1766801163247, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toBeVisible();", + "intent": "Assert that the input field remains focused by confirming the 'What needs to be done?' textbox is active and visible." + } + ] + }, + "No todo is added to the list": { + "timestamp": 1766801153675, + "actions": [] + }, + "Press Enter without typing any text": { + "timestamp": 1766801152012, + "actions": [ + { + "method": "pressKey", + "key": "Enter", + "code": "await page.keyboard.press('Enter');", + "intent": "Press Enter in the active 'What needs to be done?' textbox without entering any text, as requested." + } + ] + }, + "Todo list remains empty": { + "timestamp": 1766801154937, + "actions": [] + }, + "Add \"Test with special chars: @#$%^&*()\" todo item": { + "timestamp": 1766801172123, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Test with special chars: @#$%^&*()", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Test with special chars: @#$%^&*()');\nawait page.keyboard.press('Enter');", + "intent": "Add new todo item 'Test with special chars: @#$%^&*()' by typing into the input field and submitting." + } + ] + }, + "Todo item \"Test with special chars: @#$%^&*()\" appears in the list": { + "timestamp": 1766801179397, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Test with special chars: @#$%^&*()\"i", + "code": "await expect(page.getByText('Test with special chars: @#$%^&*()')).toBeVisible();", + "intent": "Assert that the todo item 'Test with special chars: @#$%^&*()' appears in the list." + } + ] + }, + "Add \"This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues\" todo item": { + "timestamp": 1766801188840, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues');\nawait page.keyboard.press('Enter');", + "intent": "Add the specified long todo item to test character limits and layout handling." + } + ] + }, + "Todo item \"This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues\" appears in the list": { + "timestamp": 1766801196446, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues\"i", + "code": "await expect(page.getByText('This is a very long todo item to test the character limit and see how the application handles extremely long text inputs that might break the layout or cause other issues')).toBeVisible();", + "intent": "Verify that the very long todo item is visible in the list." + } + ] + } +} \ No newline at end of file diff --git a/examples/todomvc/tests/perform/completing-todos.spec.ts b/examples/todomvc/tests/perform/completing-todos.spec.ts new file mode 100644 index 0000000000000..dd305fcd226d7 --- /dev/null +++ b/examples/todomvc/tests/perform/completing-todos.spec.ts @@ -0,0 +1,75 @@ +import { test } from '../fixtures'; + +test.use({ + agent: { + provider: 'github', + model: 'gpt-4.1', + } +}); + +test.describe('Completing Todos', () => { + + test('should complete single todo', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Click the checkbox next to "Buy groceries"`); + + await page.agent.expect(`Checkbox becomes checked`); + await page.agent.expect(`Todo text may show visual indication of completion (strikethrough or style change)`); + await page.agent.expect(`Counter shows "0 items left"`); + await page.agent.expect(`"Clear completed" button appears`); + await page.agent.expect(`Delete button becomes visible on hover`); + }); + + test('should complete multiple todos', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Add "Walk the dog" todo item`); + await page.agent.perform(`Add "Read a book" todo item`); + await page.agent.perform(`Click the checkbox next to "Buy groceries"`, { key: 'click buy groceries with multiple todos'}); + await page.agent.perform(`Click the checkbox next to "Read a book"`); + + await page.agent.expect(`Both selected todos show as completed`); + await page.agent.expect(`Counter shows "1 item left" (only "Walk the dog" remaining)`); + await page.agent.expect(`"Clear completed" button appears`); + await page.agent.expect(`One todo remains active`); + }); + + test('should uncomplete todo', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Click the checkbox to complete it`); + await page.agent.perform(`Click the checkbox again to uncomplete it`); + + await page.agent.expect(`Checkbox becomes unchecked`); + await page.agent.expect(`Todo returns to active state`); + await page.agent.expect(`Counter shows "1 item left"`); + await page.agent.expect(`"Clear completed" button disappears`); + }); + + test('should mark all as complete', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Add "Walk the dog" todo item`); + await page.agent.perform(`Add "Read a book" todo item`); + await page.agent.perform(`Click the "Mark all as complete" checkbox (chevron icon)`); + + await page.agent.expect(`All todos show as completed`); + await page.agent.expect(`All individual checkboxes are checked`); + await page.agent.expect(`"Mark all as complete" checkbox is checked`); + await page.agent.expect(`Counter shows "0 items left"`); + await page.agent.expect(`"Clear completed" button appears`); + }); + + test('should unmark all as complete', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Add "Walk the dog" todo item`); + await page.agent.perform(`Add "Read a book" todo item`); + await page.agent.perform(`Click the "Mark all as complete" checkbox to complete all`); + await page.agent.perform(`Click the "Mark all as complete" checkbox again`); + + await page.agent.expect(`All todos return to active state`); + await page.agent.expect(`All individual checkboxes are unchecked`); + await page.agent.expect(`"Mark all as complete" checkbox is unchecked`); + await page.agent.expect(`Counter shows "3 items left"`); + await page.agent.expect(`"Clear completed" button disappears`); + }); + +}); + diff --git a/examples/todomvc/tests/perform/completing-todos.spec.ts-cache.json b/examples/todomvc/tests/perform/completing-todos.spec.ts-cache.json new file mode 100644 index 0000000000000..20bf046022405 --- /dev/null +++ b/examples/todomvc/tests/perform/completing-todos.spec.ts-cache.json @@ -0,0 +1,429 @@ +{ + "\"Clear completed\" button appears": { + "timestamp": 1766802099161, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:role=button[name=\"Clear completed\"i]", + "code": "await expect(page.getByRole('button', { name: 'Clear completed' })).toBeVisible();", + "intent": "Assert that the 'Clear completed' button appears on the page as expected." + } + ] + }, + "Add \"Buy groceries\" todo item": { + "timestamp": 1766802061537, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Buy groceries", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');", + "intent": "Add 'Buy groceries' as a new todo item in the textbox for todo entry." + } + ] + }, + "Checkbox becomes checked": { + "timestamp": 1766802076871, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('checkbox', { name: 'Toggle Todo' })).toBeChecked({ checked: true });", + "intent": "Verify that the checkbox is checked to confirm expectation." + } + ] + }, + "Click the checkbox next to \"Buy groceries\"": { + "timestamp": 1766802068960, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: 'Toggle Todo' }).click();", + "intent": "Click the checkbox to mark 'Buy groceries' as complete." + } + ] + }, + "Counter shows \"0 items left\"": { + "timestamp": 1766802091992, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"0 items left\"i", + "code": "await expect(page.getByText('0 items left')).toBeVisible();", + "intent": "Verify that the counter displays '0 items left' as expected for the TodoMVC demo." + } + ] + }, + "Delete button becomes visible on hover": { + "timestamp": 1766802106350, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:role=button[name=\"Delete\"i]", + "code": "await expect(page.getByRole('button', { name: 'Delete' })).toBeVisible();", + "intent": "Verify that the Delete button becomes visible when hovering over a todo." + } + ] + }, + "Todo text may show visual indication of completion (strikethrough or style change)": { + "timestamp": 1766802084276, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries\"i", + "code": "await expect(page.getByText('Buy groceries')).toBeVisible();", + "intent": "Assert that the todo text visually indicates completion, such as strikethrough or style change. Look for the text of the completed todo." + } + ] + }, + "\"Clear completed\" button disappears": { + "timestamp": 1766802165237, + "actions": [] + }, + "Checkbox becomes unchecked": { + "timestamp": 1766802148793, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('checkbox', { name: 'Toggle Todo' })).toBeChecked({ checked: false });", + "intent": "Verify that the 'Toggle Todo' checkbox becomes unchecked as expected." + } + ] + }, + "Click the checkbox again to uncomplete it": { + "timestamp": 1766802141514, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: 'Toggle Todo' }).click();", + "intent": "Click the checked checkbox to uncomplete the todo item." + } + ] + }, + "Click the checkbox to complete it": { + "timestamp": 1766802134409, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: 'Toggle Todo' }).click();", + "intent": "Click the checkbox to complete the todo item as requested." + } + ] + }, + "Counter shows \"1 item left\"": { + "timestamp": 1766802163512, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"1 item left\"i", + "code": "await expect(page.getByText('1 item left')).toBeVisible();", + "intent": "Verify that the counter displays '1 item left' as expected." + } + ] + }, + "Todo returns to active state": { + "timestamp": 1766802156218, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"item left\"i", + "code": "await expect(page.getByText('item left')).toBeVisible();", + "intent": "Verify that a todo has returned to the active state by checking for the presence of the text 'item left' indicating active items." + } + ] + }, + "\"Mark all as complete\" checkbox is checked": { + "timestamp": 1766802233553, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('checkbox', { name: '❯Mark all as complete' })).toBeChecked({ checked: true });", + "intent": "Verify that the 'Mark all as complete' checkbox is checked to meet the expected condition." + } + ] + }, + "Add \"Read a book\" todo item": { + "timestamp": 1766802181553, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Read a book", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Read a book');\nawait page.keyboard.press('Enter');", + "intent": "Add a new todo item named 'Read a book' by typing in the input box." + } + ] + }, + "Add \"Walk the dog\" todo item": { + "timestamp": 1766802173245, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Walk the dog", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Walk the dog');\nawait page.keyboard.press('Enter');", + "intent": "Add a todo item by typing 'Walk the dog' into the new todo textbox and submitting." + } + ] + }, + "All individual checkboxes are checked": { + "timestamp": 1766802226355, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that the first individual checkbox is checked.\nVerify that the second individual checkbox is checked.\nVerify that the third individual checkbox is checked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that the first individual checkbox is checked.\nVerify that the second individual checkbox is checked.\nVerify that the third individual checkbox is checked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that the first individual checkbox is checked.\nVerify that the second individual checkbox is checked.\nVerify that the third individual checkbox is checked." + } + ] + }, + "All todos show as completed": { + "timestamp": 1766802207563, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked.\nVerify that all todos show as completed by checking their checkboxes are checked." + } + ] + }, + "Click the \"Mark all as complete\" checkbox (chevron icon)": { + "timestamp": 1766802188654, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: '❯Mark all as complete' }).click();", + "intent": "Click the 'Mark all as complete' checkbox (chevron icon) as requested." + } + ] + }, + "\"Mark all as complete\" checkbox is unchecked": { + "timestamp": 1766802294988, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('checkbox', { name: '❯Mark all as complete' })).toBeChecked({ checked: false });", + "intent": "Assert that the 'Mark all as complete' checkbox is unchecked to meet the user's expectation." + } + ] + }, + "All individual checkboxes are unchecked": { + "timestamp": 1766802287373, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Check that the checkbox for 'Buy groceries' is unchecked.\nCheck that the checkbox for 'Walk the dog' is unchecked.\nCheck that the checkbox for 'Read a book' is unchecked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Check that the checkbox for 'Buy groceries' is unchecked.\nCheck that the checkbox for 'Walk the dog' is unchecked.\nCheck that the checkbox for 'Read a book' is unchecked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Check that the checkbox for 'Buy groceries' is unchecked.\nCheck that the checkbox for 'Walk the dog' is unchecked.\nCheck that the checkbox for 'Read a book' is unchecked." + } + ] + }, + "All todos return to active state": { + "timestamp": 1766802268369, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that 'Buy groceries' todo is active, so its checkbox should be unchecked.\nAssert that 'Walk the dog' todo is active, so its checkbox should be unchecked.\nAssert that 'Read a book' todo is active, so its checkbox should be unchecked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Walk the dog\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Walk the dog' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that 'Buy groceries' todo is active, so its checkbox should be unchecked.\nAssert that 'Walk the dog' todo is active, so its checkbox should be unchecked.\nAssert that 'Read a book' todo is active, so its checkbox should be unchecked." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "false", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: false });", + "intent": "Assert that 'Buy groceries' todo is active, so its checkbox should be unchecked.\nAssert that 'Walk the dog' todo is active, so its checkbox should be unchecked.\nAssert that 'Read a book' todo is active, so its checkbox should be unchecked." + } + ] + }, + "Click the \"Mark all as complete\" checkbox again": { + "timestamp": 1766802249252, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: '❯Mark all as complete' }).click();", + "intent": "Click the 'Mark all as complete' checkbox again to toggle its state." + } + ] + }, + "Click the \"Mark all as complete\" checkbox to complete all": { + "timestamp": 1766802241875, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"❯Mark all as complete\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: '❯Mark all as complete' }).click();", + "intent": "Click the 'Mark all as complete' checkbox to complete all todos." + } + ] + }, + "Counter shows \"3 items left\"": { + "timestamp": 1766802302683, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"3 items left\"i", + "code": "await expect(page.getByText('3 items left')).toBeVisible();", + "intent": "Verify the counter displays '3 items left' as expected for remaining todos." + } + ] + }, + "Both selected todos show as completed": { + "timestamp": 1766802736844, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify 'Buy groceries' todo is shown as completed.\nVerify 'Read a book' todo is shown as completed." + }, + { + "method": "expectValue", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo')).toBeChecked({ checked: true });", + "intent": "Verify 'Buy groceries' todo is shown as completed.\nVerify 'Read a book' todo is shown as completed." + } + ] + }, + "click buy groceries with multiple todos": { + "timestamp": 1766802714652, + "actions": [ + { + "method": "click", + "selector": "internal:role=listitem >> internal:has-text=\"Buy groceries\"i >> internal:label=\"Toggle Todo\"i", + "options": {}, + "code": "await page.getByRole('listitem').filter({ hasText: 'Buy groceries' }).getByLabel('Toggle Todo').click();", + "intent": "Check the checkbox next to the 'Buy groceries' task." + } + ] + }, + "Click the checkbox next to \"Read a book\"": { + "timestamp": 1766802722604, + "actions": [ + { + "method": "click", + "selector": "internal:role=listitem >> internal:has-text=\"Read a book\"i >> internal:label=\"Toggle Todo\"i", + "options": {}, + "code": "await page.getByRole('listitem').filter({ hasText: 'Read a book' }).getByLabel('Toggle Todo').click();", + "intent": "Click the checkbox next to 'Read a book' to mark it as complete." + } + ] + }, + "Counter shows \"1 item left\" (only \"Walk the dog\" remaining)": { + "timestamp": 1766802744277, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"1 item left\"i", + "code": "await expect(page.getByText('1 item left')).toBeVisible();", + "intent": "Verify that the counter shows '1 item left' to confirm the correct number of todos remaining." + } + ] + }, + "One todo remains active": { + "timestamp": 1766802751886, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"1 item left\"i", + "code": "await expect(page.getByText('1 item left')).toBeVisible();", + "intent": "Check that only one todo is left active by confirming the text '1 item left' is visible on the page." + } + ] + } +} \ No newline at end of file diff --git a/examples/todomvc/tests/perform/editing-todos.spec.ts b/examples/todomvc/tests/perform/editing-todos.spec.ts new file mode 100644 index 0000000000000..9869dcd990c9b --- /dev/null +++ b/examples/todomvc/tests/perform/editing-todos.spec.ts @@ -0,0 +1,63 @@ +import { test } from '../fixtures'; + +test.use({ + agent: { + provider: 'github', + model: 'gpt-4.1', + } +}); + +test.describe('Editing Todos', () => { + + test('should edit todo successfully', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Double-click on the todo text "Buy groceries"`); + await page.agent.perform(`Clear the existing text`); + await page.agent.perform(`Type "Buy groceries and milk"`); + await page.agent.perform(`Press Enter`); + + await page.agent.expect(`Todo enters edit mode (input field appears)`); + await page.agent.expect(`Original text is pre-populated in the edit field`); + await page.agent.expect(`After pressing Enter, todo text updates to "Buy groceries and milk"`); + await page.agent.expect(`Todo exits edit mode`); + await page.agent.expect(`Todo remains in the same state (active/completed)`); + }); + + test('should cancel edit with escape', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Double-click on the todo text`); + await page.agent.perform(`Type "Changed text"`); + await page.agent.perform(`Press Escape key`); + + await page.agent.expect(`Todo exits edit mode`); + await page.agent.expect(`Original text "Buy groceries" is preserved`); + await page.agent.expect(`Changes are discarded`); + await page.agent.expect(`Todo remains in the same state`); + }); + + test('should delete todo by clearing text', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Double-click on the todo text`); + await page.agent.perform(`Clear all text (delete all characters)`); + await page.agent.perform(`Press Enter`); + + await page.agent.expect(`Todo is removed from the list`); + await page.agent.expect(`Counter decrements appropriately`); + await page.agent.expect(`If no todos remain, counter and controls disappear`); + }); + + test('should edit completed todo', async ({ page }) => { + await page.agent.perform(`Add "Buy groceries" todo item`); + await page.agent.perform(`Click the checkbox to complete it`); + await page.agent.perform(`Double-click on the todo text`); + await page.agent.perform(`Type "Buy groceries and milk"`); + await page.agent.perform(`Press Enter`); + + await page.agent.expect(`Todo enters edit mode`); + await page.agent.expect(`Todo text is successfully updated`); + await page.agent.expect(`Todo remains in completed state after editing`); + await page.agent.expect(`Checkbox remains checked`); + }); + +}); + diff --git a/examples/todomvc/tests/perform/editing-todos.spec.ts-cache.json b/examples/todomvc/tests/perform/editing-todos.spec.ts-cache.json new file mode 100644 index 0000000000000..cd20cd2916bb1 --- /dev/null +++ b/examples/todomvc/tests/perform/editing-todos.spec.ts-cache.json @@ -0,0 +1,249 @@ +{ + "Add \"Buy groceries\" todo item": { + "timestamp": 1766802062320, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Buy groceries", + "submit": true, + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries');\nawait page.keyboard.press('Enter');", + "intent": "Add 'Buy groceries' todo item by typing into the input box and submitting." + } + ] + }, + "After pressing Enter, todo text updates to \"Buy groceries and milk\"": { + "timestamp": 1766802109230, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries and milk\"i", + "code": "await expect(page.getByText('Buy groceries and milk')).toBeVisible();", + "intent": "Assert that after pressing Enter, the todo text updates to the expected value: 'Buy groceries and milk'." + } + ] + }, + "Clear the existing text": { + "timestamp": 1766802077991, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"Edit\"i]", + "text": "", + "code": "await page.getByRole('textbox', { name: 'Edit' }).fill('');", + "intent": "Clear the text in the 'Edit' textbox for the todo item by removing existing content." + } + ] + }, + "Double-click on the todo text \"Buy groceries\"": { + "timestamp": 1766802070894, + "actions": [ + { + "method": "click", + "selector": "internal:testid=[data-testid=\"todo-title\"s]", + "options": { + "clickCount": 2 + }, + "code": "await page.getByTestId('todo-title').click({\nclickCount: 2\n});", + "intent": "Double-click on the todo text 'Buy groceries' to enable editing." + } + ] + }, + "Original text is pre-populated in the edit field": { + "timestamp": 1766802102443, + "actions": [] + }, + "Press Enter": { + "timestamp": 1766802099251, + "actions": [ + { + "method": "pressKey", + "key": "Enter", + "code": "await page.keyboard.press('Enter');", + "intent": "Press Enter on the currently active textbox to trigger any associated action." + } + ] + }, + "Todo enters edit mode (input field appears)": { + "timestamp": 1766802101056, + "actions": [] + }, + "Todo exits edit mode": { + "timestamp": 1766802111134, + "actions": [] + }, + "Todo remains in the same state (active/completed)": { + "timestamp": 1766802118217, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries and milk\"i", + "code": "await expect(page.getByText('Buy groceries and milk')).toBeVisible();", + "intent": "Verify the todo remains in the same state by asserting its visible text content." + } + ] + }, + "Type \"Buy groceries and milk\"": { + "timestamp": 1766802092021, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "text": "Buy groceries and milk", + "code": "await page.getByRole('textbox', { name: 'What needs to be done?' }).fill('Buy groceries and milk');", + "intent": "Type 'Buy groceries and milk' into the 'What needs to be done?' input to add a new todo item." + }, + { + "method": "pressKey", + "key": "Enter", + "code": "await page.keyboard.press('Enter');", + "intent": "Submit the new todo item 'Buy groceries and milk' by pressing Enter in the input field." + } + ] + }, + "Changes are discarded": { + "timestamp": 1766802157281, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries\"i", + "code": "await expect(page.getByText('Buy groceries')).toBeVisible();", + "intent": "Verify that the text indicating changes are discarded (original state visible) is present." + } + ] + }, + "Double-click on the todo text": { + "timestamp": 1766802128825, + "actions": [ + { + "method": "click", + "selector": "internal:testid=[data-testid=\"todo-title\"s]", + "options": { + "clickCount": 2 + }, + "code": "await page.getByTestId('todo-title').click({\nclickCount: 2\n});", + "intent": "Double-click on the todo text 'Buy groceries' as requested." + } + ] + }, + "Original text \"Buy groceries\" is preserved": { + "timestamp": 1766802150160, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries\"i", + "code": "await expect(page.getByText('Buy groceries')).toBeVisible();", + "intent": "Verify that the original text 'Buy groceries' is still present and preserved on the page as expected." + } + ] + }, + "Press Escape key": { + "timestamp": 1766802143007, + "actions": [ + { + "method": "pressKey", + "key": "Escape", + "code": "await page.keyboard.press('Escape');", + "intent": "Press Escape key to cancel or close any active inputs or dialogs, such as the currently active edit textbox." + } + ] + }, + "Todo remains in the same state": { + "timestamp": 1766802164707, + "actions": [ + { + "method": "expectAria", + "template": "- list:\n - listitem: Buy groceries", + "code": "await expect(page.locator('body')).toMatchAria(`\n- list:\n - listitem: Buy groceries\n`);", + "intent": "Verify the todo list remains in the same state with one list item: 'Buy groceries'." + } + ] + }, + "Type \"Changed text\"": { + "timestamp": 1766802135821, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"Edit\"i]", + "text": "Changed text", + "code": "await page.getByRole('textbox', { name: 'Edit' }).fill('Changed text');", + "intent": "Edit the existing todo item and change its text to 'Changed text'." + } + ] + }, + "Clear all text (delete all characters)": { + "timestamp": 1766802173007, + "actions": [ + { + "method": "fill", + "selector": "internal:role=textbox[name=\"Edit\"i]", + "text": "", + "code": "await page.getByRole('textbox', { name: 'Edit' }).fill('');", + "intent": "Clear the text from the active Edit textbox by deleting all characters." + } + ] + }, + "Counter decrements appropriately": { + "timestamp": 1766802177914, + "actions": [] + }, + "If no todos remain, counter and controls disappear": { + "timestamp": 1766802185553, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:role=textbox[name=\"What needs to be done?\"i]", + "code": "await expect(page.getByRole('textbox', { name: 'What needs to be done?' })).toBeVisible();", + "intent": "Assert the controls and counter are not visible when no todos remain.\nVerify that no counter or controls are visible due to absence of todos." + } + ] + }, + "Todo is removed from the list": { + "timestamp": 1766802174741, + "actions": [] + }, + "Checkbox remains checked": { + "timestamp": 1766802212342, + "actions": [ + { + "method": "expectValue", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i] >> nth=0", + "type": "checkbox", + "value": "true", + "code": "await expect(page.getByRole('checkbox', { name: 'Toggle Todo' }).first()).toBeChecked({ checked: true });", + "intent": "Verify that the first todo item's checkbox remains checked to meet expectation." + } + ] + }, + "Click the checkbox to complete it": { + "timestamp": 1766802193838, + "actions": [ + { + "method": "click", + "selector": "internal:role=checkbox[name=\"Toggle Todo\"i]", + "options": {}, + "code": "await page.getByRole('checkbox', { name: 'Toggle Todo' }).click();", + "intent": "Click the checkbox to complete the todo item." + } + ] + }, + "Todo enters edit mode": { + "timestamp": 1766802195745, + "actions": [] + }, + "Todo remains in completed state after editing": { + "timestamp": 1766802204977, + "actions": [] + }, + "Todo text is successfully updated": { + "timestamp": 1766802202733, + "actions": [ + { + "method": "expectVisible", + "selector": "internal:text=\"Buy groceries and milk\"i", + "code": "await expect(page.getByText('Buy groceries and milk')).toBeVisible();", + "intent": "Verify that the todo text has been successfully updated from 'Buy groceries' to 'Buy groceries and milk'." + } + ] + } +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1c82ec5b7c2f6..3ebe7b10a3f57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.11", + "@lowire/loop": "^0.0.12", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", @@ -1064,9 +1064,9 @@ } }, "node_modules/@lowire/loop": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz", - "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==", + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.12.tgz", + "integrity": "sha512-WUAmIm6qzGuku0f/6t613nwhaS9dYhOk1CmceENxRm+AdFVgMVw60bj5QwtKmwacPjVO5TkIY895YLvzOfFf+w==", "dev": true, "license": "Apache-2.0", "engines": { diff --git a/package.json b/package.json index 5629dff786905..96be334260358 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "test": "playwright test --config=tests/library/playwright.config.ts", "test-mcp": "playwright test --config=tests/mcp/playwright.config.ts", "ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome", - "etest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=eval", "eslint": "eslint --cache", "tsc": "tsc -p . && tsc -p packages/html-reporter/", "doc": "node utils/doclint/cli.js", @@ -56,7 +55,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.11", + "@lowire/loop": "^0.0.12", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts index 5bf24257a571a..6f7348115baae 100644 --- a/packages/injected/src/ariaSnapshot.ts +++ b/packages/injected/src/ariaSnapshot.ts @@ -16,10 +16,10 @@ import * as aria from '@isomorphic/ariaSnapshot'; import { escapeRegExp, longestCommonSubstring, normalizeWhiteSpace } from '@isomorphic/stringUtils'; +import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from '@isomorphic/yaml'; import { computeBox, getElementComputedStyle, isElementVisible } from './domUtils'; import * as roleUtils from './roleUtils'; -import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from './yaml'; export type AriaSnapshot = { root: aria.AriaNode; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 002616f81a0df..201562bf07c6e 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -1018,24 +1018,6 @@ export interface Page { */ behavior?: 'wait'|'ignoreErrors'|'default' }): Promise; - - /** - * Extract information from the page using the agentic loop, return it in a given Zod format. - * - * **Usage** - * - * ```js - * await page.extract('List of items in the cart', z.object({ - * title: z.string().describe('Item title to extract'), - * price: z.string().describe('Item price to extract'), - * }).array()); - * ``` - * - * @param query Task to perform using agentic loop. - * @param schema - * @param options - */ - extract(query: string, schema: Schema): Promise>; /** * Emitted when the agent makes a turn. */ @@ -3909,45 +3891,6 @@ export interface Page { width?: string|number; }): Promise; - /** - * Perform action using agentic loop. - * - * **Usage** - * - * ```js - * await page.perform('Click submit button'); - * ``` - * - * @param task Task to perform using agentic loop. - * @param options - */ - perform(task: string, options?: { - /** - * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally - * with the `task` as a key. This option allows controlling the cache key explicitly. - */ - key?: string; - - /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. - */ - maxTokens?: number; - - /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. - */ - maxTurns?: number; - }): Promise<{ - usage: { - turns: number; - - inputTokens: number; - - outputTokens: number; - }; - }>; - /** * **NOTE** Use locator-based [locator.press(key[, options])](https://playwright.dev/docs/api/class-locator#locator-press) * instead. Read more about [locators](https://playwright.dev/docs/locators). @@ -5360,6 +5303,8 @@ export interface Page { */ workers(): Array; + agent: PageAgent; + /** * Playwright has ability to mock clock and passage of time. */ @@ -5391,6 +5336,98 @@ export interface Page { [Symbol.asyncDispose](): Promise; } +/** + * + */ +export interface PageAgent { + /** + * Extract information from the page using the agentic loop, return it in a given Zod format. + * + * **Usage** + * + * ```js + * await page.agent.extract('List of items in the cart', z.object({ + * title: z.string().describe('Item title to extract'), + * price: z.string().describe('Item price to extract'), + * }).array()); + * ``` + * + * @param query Task to perform using agentic loop. + * @param schema + * @param options + */ + extract(query: string, schema: Schema): Promise>; + /** + * Expect certain condition to be met. + * + * **Usage** + * + * ```js + * await page.agent.expect('"0 items" to be reported'); + * ``` + * + * @param expectation Expectation to assert. + * @param options + */ + expect(expectation: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + key?: string; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + */ + maxTurns?: number; + }): Promise; + + /** + * Perform action using agentic loop. + * + * **Usage** + * + * ```js + * await page.agent.perform('Click submit button'); + * ``` + * + * @param task Task to perform using agentic loop. + * @param options + */ + perform(task: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + key?: string; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + */ + maxTurns?: number; + }): Promise<{ + usage: { + turns: number; + + inputTokens: number; + + outputTokens: number; + }; + }>; +} + /** * At every point of time, page exposes its current frame tree via the * [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and @@ -22227,8 +22264,7 @@ export interface BrowserContextOptions { acceptDownloads?: boolean; /** - * Agent settings for [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and - * [page.extract(query, schema[, options])](https://playwright.dev/docs/api/class-page#page-extract). + * Agent settings for [page.agent](https://playwright.dev/docs/api/class-page#page-agent). */ agent?: { /** diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt index 7ed4413e0ba3c..55b8910bbba25 100644 --- a/packages/playwright-core/ThirdPartyNotices.txt +++ b/packages/playwright-core/ThirdPartyNotices.txt @@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. -- @lowire/loop@0.0.11 (https://github.com/pavelfeldman/lowire) +- @lowire/loop@0.0.12 (https://github.com/pavelfeldman/lowire) - @modelcontextprotocol/sdk@1.24.2 (https://github.com/modelcontextprotocol/typescript-sdk) - accepts@2.0.0 (https://github.com/jshttp/accepts) - agent-base@7.1.4 (https://github.com/TooTallNate/proxy-agents) @@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina - zod-to-json-schema@3.25.0 (https://github.com/StefanTerdell/zod-to-json-schema) - zod@3.25.76 (https://github.com/colinhacks/zod) -%% @lowire/loop@0.0.11 NOTICES AND INFORMATION BEGIN HERE +%% @lowire/loop@0.0.12 NOTICES AND INFORMATION BEGIN HERE ========================================= Apache License Version 2.0, January 2004 @@ -339,7 +339,7 @@ Apache License See the License for the specific language governing permissions and limitations under the License. ========================================= -END OF @lowire/loop@0.0.11 AND INFORMATION +END OF @lowire/loop@0.0.12 AND INFORMATION %% @modelcontextprotocol/sdk@1.24.2 NOTICES AND INFORMATION BEGIN HERE ========================================= diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json index 2754245dd72fa..d230205c39bc5 100644 --- a/packages/playwright-core/bundles/mcp/package-lock.json +++ b/packages/playwright-core/bundles/mcp/package-lock.json @@ -8,16 +8,16 @@ "name": "mcp-bundle", "version": "0.0.1", "dependencies": { - "@lowire/loop": "^0.0.11", + "@lowire/loop": "^0.0.12", "@modelcontextprotocol/sdk": "^1.24.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" } }, "node_modules/@lowire/loop": { - "version": "0.0.11", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz", - "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==", + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.12.tgz", + "integrity": "sha512-WUAmIm6qzGuku0f/6t613nwhaS9dYhOk1CmceENxRm+AdFVgMVw60bj5QwtKmwacPjVO5TkIY895YLvzOfFf+w==", "license": "Apache-2.0", "engines": { "node": ">=20" diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json index 2c6d12e6f0022..c5e8c16d08a37 100644 --- a/packages/playwright-core/bundles/mcp/package.json +++ b/packages/playwright-core/bundles/mcp/package.json @@ -3,7 +3,7 @@ "version": "0.0.1", "private": true, "dependencies": { - "@lowire/loop": "^0.0.11", + "@lowire/loop": "^0.0.12", "@modelcontextprotocol/sdk": "^1.24.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" diff --git a/packages/playwright-core/src/client/api.ts b/packages/playwright-core/src/client/api.ts index a7dab63f1856f..e4b0abb923f90 100644 --- a/packages/playwright-core/src/client/api.ts +++ b/packages/playwright-core/src/client/api.ts @@ -36,6 +36,7 @@ export { JSHandle } from './jsHandle'; export { Request, Response, Route, WebSocket, WebSocketRoute } from './network'; export { APIRequest, APIRequestContext, APIResponse } from './fetch'; export { Page } from './page'; +export { PageAgent } from './pageAgent'; export { Selectors } from './selectors'; export { Tracing } from './tracing'; export { Video } from './video'; diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 1c3cbe9e22b14..bfab6af21de89 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -41,6 +41,7 @@ import { urlMatches, urlMatchesEqual } from '../utils/isomorphic/urlMatch'; import { LongStandingScope } from '../utils/isomorphic/manualPromise'; import { isObject, isRegExp, isString } from '../utils/isomorphic/rtti'; import { ConsoleMessage } from './consoleMessage'; +import { PageAgent } from './pageAgent'; import type { BrowserContext } from './browserContext'; import type { Clock } from './clock'; @@ -54,7 +55,6 @@ import type * as api from '../../types/types'; import type { ByRoleOptions } from '../utils/isomorphic/locatorUtils'; import type { URLMatch } from '../utils/isomorphic/urlMatch'; import type * as channels from '@protocol/channels'; -import type z from 'zod'; type PDFOptions = Omit & { width?: string | number, @@ -89,6 +89,7 @@ export class Page extends ChannelOwner implements api.Page _routes: RouteHandler[] = []; _webSocketRoutes: WebSocketRouteHandler[] = []; + readonly agent: PageAgent; readonly coverage: Coverage; readonly keyboard: Keyboard; readonly mouse: Mouse; @@ -121,6 +122,7 @@ export class Page extends ChannelOwner implements api.Page this._browserContext = parent as unknown as BrowserContext; this._timeoutSettings = new TimeoutSettings(this._platform, this._browserContext._timeoutSettings); + this.agent = new PageAgent(this); this.keyboard = new Keyboard(this); this.mouse = new Mouse(this); this.request = this._browserContext.request; @@ -847,16 +849,6 @@ export class Page extends ChannelOwner implements api.Page return result.pdf; } - async perform(task: string, options: { key?: string, maxTokens?: number, maxTurns?: number } = {}) { - const result = await this._channel.perform({ task, ...options }); - return { usage: { ...result } }; - } - - async extract(query: string, schema: Schema, options: { maxTokens?: number, maxTurns?: number } = {}): Promise> { - const { result, ...usage } = await this._channel.extract({ query, schema: this._platform.zodToJsonSchema(schema), ...options }); - return { result, usage }; - } - async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> { return await this._channel.snapshotForAI({ timeout: this._timeoutSettings.timeout(options), track: options.track }); } diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts new file mode 100644 index 0000000000000..f8c3e317ea750 --- /dev/null +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -0,0 +1,42 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type * as api from '../../types/types'; +import type { Page } from './page'; +import type z from 'zod'; + +export class PageAgent implements api.PageAgent { + private _page: Page; + + constructor(page: Page) { + this._page = page; + } + + async expect(expectation: string, options: { maxTokens?: number, maxTurns?: number } = {}) { + await this._page._channel.agentExpect({ expectation, ...options }); + } + + async perform(task: string, options: { key?: string, maxTokens?: number, maxTurns?: number } = {}) { + const result = await this._page._channel.agentPerform({ task, ...options }); + return { usage: { ...result } }; + } + + async extract(query: string, schema: Schema, options: { maxTokens?: number, maxTurns?: number } = {}): Promise> { + const { result, ...usage } = await this._page._channel.agentExtract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); + return { result, usage }; + } +} diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index f2fd5f3851980..c90fbdaa493a0 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1527,23 +1527,32 @@ scheme.PageUpdateSubscriptionParams = tObject({ enabled: tBoolean, }); scheme.PageUpdateSubscriptionResult = tOptional(tObject({})); -scheme.PagePerformParams = tObject({ +scheme.PageAgentPerformParams = tObject({ task: tString, key: tOptional(tString), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), }); -scheme.PagePerformResult = tObject({ +scheme.PageAgentPerformResult = tObject({ turns: tInt, inputTokens: tInt, outputTokens: tInt, }); -scheme.PageExtractParams = tObject({ +scheme.PageAgentExpectParams = tObject({ + expectation: tString, + key: tOptional(tString), + maxTurns: tOptional(tInt), + maxTokens: tOptional(tInt), +}); +scheme.PageAgentExpectResult = tOptional(tObject({})); +scheme.PageAgentExtractParams = tObject({ query: tString, schema: tAny, + key: tOptional(tString), maxTurns: tOptional(tInt), + maxTokens: tOptional(tInt), }); -scheme.PageExtractResult = tObject({ +scheme.PageAgentExtractResult = tObject({ result: tAny, turns: tInt, inputTokens: tInt, diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts index efe7f4d072ec7..3ac5d3707ff74 100644 --- a/packages/playwright-core/src/server/agent/actionRunner.ts +++ b/packages/playwright-core/src/server/agent/actionRunner.ts @@ -16,7 +16,9 @@ import { serializeExpectedTextValues } from '../utils/expectUtils'; import { monotonicTime } from '../../utils/isomorphic/time'; +import { parseAriaSnapshotUnsafe } from '../../utils/isomorphic/ariaSnapshot'; import { ProgressController } from '../progress'; +import { yaml } from '../../utilsBundle'; import type * as actions from './actions'; import type { Page } from '../page'; @@ -75,7 +77,7 @@ async function innerRunAction(progress: Progress, page: Page, action: actions.Ac break; case 'expectVisible': { const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false }); - if (result.errorMessage) + if (!result.matches) throw new Error(result.errorMessage); break; } @@ -94,6 +96,13 @@ async function innerRunAction(progress: Progress, page: Page, action: actions.Ac throw new Error(result.errorMessage); break; } + case 'expectAria': { + const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template); + const result = await frame.expect(progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: false }); + if (!result.matches) + throw new Error(result.errorMessage); + break; + } } } @@ -110,6 +119,7 @@ export function generateActionTimeout(action: actions.Action): number { return 5000; case 'expectVisible': case 'expectValue': + case 'expectAria': return 1; // one shot } } @@ -127,6 +137,7 @@ export function performActionTimeout(action: actions.Action): number { return 0; // no timeout case 'expectVisible': case 'expectValue': + case 'expectAria': return 5000; // default expect timeout. } } diff --git a/packages/playwright-core/src/server/agent/actions.ts b/packages/playwright-core/src/server/agent/actions.ts index 2c31cb0e62710..000573f5c1225 100644 --- a/packages/playwright-core/src/server/agent/actions.ts +++ b/packages/playwright-core/src/server/agent/actions.ts @@ -77,7 +77,24 @@ export type ExpectValue = { value: string; }; -export type Action = ClickAction | DragAction | HoverAction | SelectOptionAction | PressAction | PressSequentiallyAction | FillAction | SetChecked | ExpectVisible | ExpectValue; +export type ExpectAria = { + method: 'expectAria'; + template: string; +}; + +export type Action = + | ClickAction + | DragAction + | HoverAction + | SelectOptionAction + | PressAction + | PressSequentiallyAction + | FillAction + | SetChecked + | ExpectVisible + | ExpectValue + | ExpectAria; + export type ActionWithCode = Action & { code: string; intent?: string; diff --git a/packages/playwright-core/src/server/agent/codegen.ts b/packages/playwright-core/src/server/agent/codegen.ts index 187ff70d04724..d0bc4d9b71c52 100644 --- a/packages/playwright-core/src/server/agent/codegen.ts +++ b/packages/playwright-core/src/server/agent/codegen.ts @@ -15,7 +15,7 @@ */ import { asLocator } from '../../utils/isomorphic/locatorGenerators'; -import { escapeWithQuotes, formatObjectOrVoid } from '../../utils/isomorphic/stringUtils'; +import { escapeTemplateString, escapeWithQuotes, formatObjectOrVoid } from '../../utils/isomorphic/stringUtils'; import type * as actions from './actions'; import type { Language } from '../../utils/isomorphic/locatorGenerators'; @@ -73,6 +73,9 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action return `await expect(page.${locator}).toBeChecked({ checked: ${action.value === 'true'} });`; return `await expect(page.${locator}).toHaveValue(${escapeWithQuotes(action.value)});`; } + case 'expectAria': { + return `await expect(page.locator('body')).toMatchAria(\`\n${escapeTemplateString(action.template)}\n\`);`; + } } // @ts-expect-error throw new Error('Unknown action ' + action.method); diff --git a/packages/playwright-core/src/server/agent/context.ts b/packages/playwright-core/src/server/agent/context.ts index 2102af4ac21e9..684b75097e5a4 100644 --- a/packages/playwright-core/src/server/agent/context.ts +++ b/packages/playwright-core/src/server/agent/context.ts @@ -25,7 +25,7 @@ import type { Page } from '../page'; import type { Progress } from '../progress'; import type { BrowserContextOptions } from '../types'; import type { Language } from '../../utils/isomorphic/locatorGenerators.ts'; -import type { ToolDefinition } from './tools'; +import type { ToolDefinition } from './tool'; type AgentOptions = BrowserContextOptions['agent']; diff --git a/packages/playwright-core/src/server/agent/expectTools.ts b/packages/playwright-core/src/server/agent/expectTools.ts new file mode 100644 index 0000000000000..cf5309ecb8900 --- /dev/null +++ b/packages/playwright-core/src/server/agent/expectTools.ts @@ -0,0 +1,113 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from '../../mcpBundle'; +import { getByRoleSelector, getByTextSelector } from '../../utils/isomorphic/locatorUtils'; +import { yamlEscapeValueIfNeeded } from '../../utils/isomorphic/yaml'; +import { defineTool } from './tool'; + +import type { ToolDefinition } from './tool'; + +const expectVisible = defineTool({ + schema: { + name: 'browser_expect_visible', + title: 'Expect element visible', + description: 'Expect element is visible on the page', + inputSchema: z.object({ + role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'), + accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'), + }), + }, + + handle: async (context, params) => { + return await context.runActionAndWait({ + method: 'expectVisible', + selector: getByRoleSelector(params.role, { name: params.accessibleName }), + }); + }, +}); + +const expectVisibleText = defineTool({ + schema: { + name: 'browser_expect_visible_text', + title: 'Expect text visible', + description: `Expect text is visible on the page. Prefer ${expectVisible.schema.name} if possible.`, + inputSchema: z.object({ + text: z.string().describe('TEXT to expect. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'), + }), + }, + + handle: async (context, params) => { + return await context.runActionAndWait({ + method: 'expectVisible', + selector: getByTextSelector(params.text), + }); + }, +}); + +const expectValue = defineTool({ + schema: { + name: 'browser_expect_value', + title: 'Expect value', + description: 'Expect element value', + inputSchema: z.object({ + type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'), + element: z.string().describe('Human-readable element description'), + ref: z.string().describe('Exact target element reference from the page snapshot'), + value: z.string().describe('Value to expect. For checkbox, use "true" or "false".'), + }), + }, + + handle: async (context, params) => { + const [selector] = await context.refSelectors([{ ref: params.ref, element: params.element }]); + return await context.runActionAndWait({ + method: 'expectValue', + selector, + type: params.type, + value: params.value, + }); + }, +}); + +const expectList = defineTool({ + schema: { + name: 'browser_expect_list_visible', + title: 'Expect list visible', + description: 'Expect list is visible on the page, ensures items are present in the element in the exact order', + inputSchema: z.object({ + listRole: z.string().describe('Aria role of the list element as in the snapshot'), + listAccessibleName: z.string().optional().describe('Accessible name of the list element as in the snapshot'), + itemRole: z.string().describe('Aria role of the list items as in the snapshot, should all be the same'), + items: z.array(z.string().describe('Text to look for in the list item, can be either from accessible name of self / nested text content')), + }), + }, + + handle: async (context, params) => { + const template = `- ${params.listRole}: +${params.items.map(item => ` - ${params.itemRole}: ${yamlEscapeValueIfNeeded(item)}`).join('\n')}`; + return await context.runActionAndWait({ + method: 'expectAria', + template, + }); + }, +}); + +export default [ + expectVisible, + expectVisibleText, + expectValue, + expectList, +] as ToolDefinition[]; diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts similarity index 68% rename from packages/playwright-core/src/server/agent/agent.ts rename to packages/playwright-core/src/server/agent/pageAgent.ts index 4147ef1602090..bb5e275b6cd1e 100644 --- a/packages/playwright-core/src/server/agent/agent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -16,17 +16,20 @@ import fs from 'fs'; -import { toolsForLoop } from './backend'; +import { toolsForLoop } from './tool'; import { debug } from '../../utilsBundle'; import { Loop } from '../../mcpBundle'; import { runAction } from './actionRunner'; import { Context } from './context'; import { Page } from '../page'; +import performTools from './performTools'; +import expectTools from './expectTools'; import type { Progress } from '../progress'; import type * as channels from '@protocol/channels'; import type * as loopTypes from '@lowire/loop'; import type * as actions from './actions'; +import type { ToolDefinition } from './tool'; type Usage = { turns: number, @@ -34,33 +37,69 @@ type Usage = { outputTokens: number, }; -export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise { +export async function pageAgentPerform(progress: Progress, page: Page, options: channels.PageAgentPerformParams): Promise { const context = new Context(progress, page); - if (await cachedPerform(progress, context, options)) + const cacheKey = (options.key ?? options.task).trim(); + if (await cachedPerform(progress, context, cacheKey)) + return { turns: 0, inputTokens: 0, outputTokens: 0 }; + + const task = ` +### Instructions +- Perform the following task on the page. +- Your reply should be a tool call that performs action the page". + +### Task +${options.task} +`; + + const { usage } = await runLoop(progress, context, performTools, task, undefined, options); + await updateCache(context, cacheKey); + return usage; +} + +export async function pageAgentExpect(progress: Progress, page: Page, options: channels.PageAgentExpectParams): Promise { + const context = new Context(progress, page); + + const cacheKey = (options.key ?? options.expectation).trim(); + if (await cachedPerform(progress, context, cacheKey)) return { turns: 0, inputTokens: 0, outputTokens: 0 }; - const { usage } = await perform(progress, context, options.task, undefined, options); - await updateCache(context, options); + const task = ` +### Instructions +- Call one of the "browser_expect_*" tools to verify / assert the condition. +- You can call exactly one tool and it can't be report_results, must be one of the assertion tools. + +### Expectation +${options.expectation} +`; + + const { usage } = await runLoop(progress, context, expectTools, task, undefined, options); + await updateCache(context, cacheKey); return usage; } -export async function pageExtract(progress: Progress, page: Page, options: channels.PageExtractParams): Promise<{ +export async function pageAgentExtract(progress: Progress, page: Page, options: channels.PageAgentExtractParams): Promise<{ result: any, usage: Usage }> { + const context = new Context(progress, page); + const task = ` ### Instructions Extract the following information from the page. Do not perform any actions, just extract the information. ### Query ${options.query}`; - const { result, usage } = await perform(progress, context, task, options.schema, options); + const { result, usage } = await runLoop(progress, context, [], task, options.schema, options); return { result, usage }; } -async function perform(progress: Progress, context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number, maxTokens?: number } = {}): Promise<{ +async function runLoop(progress: Progress, context: Context, toolDefinitions: ToolDefinition[], userTask: string, resultSchema: loopTypes.Schema | undefined, options: { + maxTurns?: number; + maxTokens?: number; +}): Promise<{ result: any, usage: Usage }> { @@ -70,7 +109,7 @@ async function perform(progress: Progress, context: Context, userTask: string, r throw new Error(`This action requires the agent provider and model to be set on the browser context`); const { full } = await page.snapshotForAI(progress); - const { tools, callTool } = toolsForLoop(context); + const { tools, callTool } = toolsForLoop(context, toolDefinitions, { resultSchema }); page.emit(Page.Events.AgentTurn, { role: 'user', message: userTask }); @@ -104,8 +143,6 @@ async function perform(progress: Progress, context: Context, userTask: string, r onAfterToolCall: ({ toolCall }) => { const suffix = toolCall.result?.isError ? 'failed' : 'succeeded'; page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${toolCall.name}" ${suffix}` }); - if (toolCall.arguments.thatShouldBeIt) - return 'break'; return 'continue'; }, onToolCallError: ({ toolCall, error }) => { @@ -121,7 +158,7 @@ async function perform(progress: Progress, context: Context, userTask: string, r ${full} `; - const { result, usage } = await loop.run(task, { resultSchema }); + const { result, usage } = await loop.run(task); return { result, usage: { @@ -137,12 +174,11 @@ type CachedActions = Record; -async function cachedPerform(progress: Progress, context: Context, options: channels.PagePerformParams): Promise { +async function cachedPerform(progress: Progress, context: Context, cacheKey: string): Promise { if (!context.options?.cacheFile) return false; - const cache = await cachedActions(context, context.options?.cacheFile); - const cacheKey = (options.key ?? options.task).trim(); + const cache = await cachedActions(context.options?.cacheFile); const entry = cache.actions[cacheKey]; if (!entry) return false; @@ -152,12 +188,11 @@ async function cachedPerform(progress: Progress, context: Context, options: chan return true; } -async function updateCache(context: Context, options: channels.PagePerformParams) { +async function updateCache(context: Context, cacheKey: string) { const cacheFile = context.options?.cacheFile; const cacheOutFile = context.options?.cacheOutFile; - const cache = cacheFile ? await cachedActions(context, cacheFile) : { actions: {}, newActions: {} }; - const cacheKey = (options.key ?? options.task).trim(); + const cache = cacheFile ? await cachedActions(cacheFile) : { actions: {}, newActions: {} }; const newEntry = { timestamp: Date.now(), actions: context.actions, @@ -181,14 +216,14 @@ type Cache = { newActions: CachedActions; }; -async function cachedActions(context: Context, cacheFile: string): Promise { - let cache = (context as any)[agentCacheSymbol] as Cache | undefined; +const allCaches = new Map(); + +async function cachedActions(cacheFile: string): Promise { + let cache = allCaches.get(cacheFile); if (!cache) { const actions = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions; cache = { actions, newActions: {} }; - (context as any)[agentCacheSymbol] = cache; + allCaches.set(cacheFile, cache); } return cache; } - -const agentCacheSymbol = Symbol('agentCache'); diff --git a/packages/playwright-core/src/server/agent/tools.ts b/packages/playwright-core/src/server/agent/performTools.ts similarity index 70% rename from packages/playwright-core/src/server/agent/tools.ts rename to packages/playwright-core/src/server/agent/performTools.ts index f14671aef06ee..8563099edb0cc 100644 --- a/packages/playwright-core/src/server/agent/tools.ts +++ b/packages/playwright-core/src/server/agent/performTools.ts @@ -15,37 +15,17 @@ */ import { z } from '../../mcpBundle'; -import { getByRoleSelector, getByTextSelector } from '../../utils/isomorphic/locatorUtils'; +import { defineTool } from './tool'; -import type zod from 'zod'; -import type * as loopTypes from '@lowire/loop'; import type * as actions from './actions'; -import type { Context } from './context'; - -type ToolSchema = Omit & { - title: string; - inputSchema: Input; -}; - -export type ToolDefinition = { - schema: ToolSchema; - handle: (context: Context, params: zod.output) => Promise; -}; - -function defineTool(tool: ToolDefinition): ToolDefinition { - return tool; -} - -const baseSchema = z.object({ - thatShouldBeIt: z.boolean().describe('Indicates that this tool call is sufficient to complete the task. If false, the task will continue with the next tool call'), -}); +import type { ToolDefinition } from './tool'; const snapshot = defineTool({ schema: { name: 'browser_snapshot', title: 'Page snapshot', description: 'Capture accessibility snapshot of the current page, this is better than screenshot', - inputSchema: baseSchema, + inputSchema: z.object({}), }, handle: async (context, params) => { @@ -53,7 +33,7 @@ const snapshot = defineTool({ }, }); -const elementSchema = baseSchema.extend({ +const elementSchema = z.object({ element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'), ref: z.string().describe('Exact target element reference from the page snapshot'), }); @@ -91,7 +71,7 @@ const drag = defineTool({ name: 'browser_drag', title: 'Drag mouse', description: 'Perform drag and drop between two elements', - inputSchema: baseSchema.extend({ + inputSchema: z.object({ startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'), startRef: z.string().describe('Exact source element reference from the page snapshot'), endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'), @@ -216,7 +196,7 @@ const fillForm = defineTool({ name: 'browser_fill_form', title: 'Fill form', description: 'Fill multiple form fields. Always use this tool when you can fill more than one field at a time.', - inputSchema: baseSchema.extend({ + inputSchema: z.object({ fields: z.array(z.object({ name: z.string().describe('Human-readable field name'), type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the field'), @@ -254,67 +234,6 @@ const fillForm = defineTool({ }, }); -const expectVisible = defineTool({ - schema: { - name: 'browser_expect_visible', - title: 'Expect element visible', - description: 'Expect element is visible on the page', - inputSchema: baseSchema.extend({ - role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'), - accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'), - }), - }, - - handle: async (context, params) => { - return await context.runActionAndWait({ - method: 'expectVisible', - selector: getByRoleSelector(params.role, { name: params.accessibleName }), - }); - }, -}); - -const expectVisibleText = defineTool({ - schema: { - name: 'browser_expect_visible_text', - title: 'Expect text visible', - description: `Expect text is visible on the page. Prefer ${expectVisible.schema.name} if possible.`, - inputSchema: baseSchema.extend({ - text: z.string().describe('TEXT to expect. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'), - }), - }, - - handle: async (context, params) => { - return await context.runActionAndWait({ - method: 'expectVisible', - selector: getByTextSelector(params.text), - }); - }, -}); - -const expectValue = defineTool({ - schema: { - name: 'browser_expect_value', - title: 'Expect value', - description: 'Expect element value', - inputSchema: baseSchema.extend({ - type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'), - element: z.string().describe('Human-readable element description'), - ref: z.string().describe('Exact target element reference from the page snapshot'), - value: z.string().describe('Value to expect. For checkbox, use "true" or "false".'), - }), - }, - - handle: async (context, params) => { - const [selector] = await context.refSelectors([{ ref: params.ref, element: params.element }]); - return await context.runActionAndWait({ - method: 'expectValue', - selector, - type: params.type, - value: params.value, - }); - }, -}); - export default [ snapshot, click, @@ -324,7 +243,4 @@ export default [ pressKey, type, fillForm, - expectVisible, - expectVisibleText, - expectValue, ] as ToolDefinition[]; diff --git a/packages/playwright-core/src/server/agent/backend.ts b/packages/playwright-core/src/server/agent/tool.ts similarity index 62% rename from packages/playwright-core/src/server/agent/backend.ts rename to packages/playwright-core/src/server/agent/tool.ts index bccd7035e4d0d..365cb865badca 100644 --- a/packages/playwright-core/src/server/agent/backend.ts +++ b/packages/playwright-core/src/server/agent/tool.ts @@ -14,13 +14,27 @@ * limitations under the License. */ -import toolDefinitions from './tools'; import { zodToJsonSchema } from '../../mcpBundle'; +import type zod from 'zod'; import type * as loopTypes from '@lowire/loop'; import type { Context } from './context'; -export function toolsForLoop(context: Context): { tools: loopTypes.Tool[], callTool: loopTypes.ToolCallback } { +export type ToolSchema = Omit & { + title: string; + inputSchema: Input; +}; + +export type ToolDefinition = { + schema: ToolSchema; + handle: (context: Context, params: zod.output) => Promise; +}; + +export function defineTool(tool: ToolDefinition): ToolDefinition { + return tool; +} + +export function toolsForLoop(context: Context, toolDefinitions: ToolDefinition[], options: { resultSchema?: loopTypes.Schema } = {}): { tools: loopTypes.Tool[], callTool: loopTypes.ToolCallback } { const tools = toolDefinitions.map(tool => { const result: loopTypes.Tool = { name: tool.schema.name, @@ -29,9 +43,23 @@ export function toolsForLoop(context: Context): { tools: loopTypes.Tool[], callT }; return result; }); + if (options.resultSchema) { + tools.push({ + name: 'report_result', + description: 'Report the result of the task.', + inputSchema: options.resultSchema, + }); + } const callTool: loopTypes.ToolCallback = async params => { const intent = params.arguments._meta?.['dev.lowire/intent']; + if (params.name === 'report_result') { + return { + content: [{ type: 'text', text: JSON.stringify(params.arguments) }], + isError: false, + }; + } + const tool = toolDefinitions.find(t => t.schema.name === params.name); if (!tool) { return { diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts index 1ba01607c50c2..b59761f383803 100644 --- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts @@ -27,7 +27,7 @@ import { RouteDispatcher, WebSocketDispatcher } from './networkDispatchers'; import { WebSocketRouteDispatcher } from './webSocketRouteDispatcher'; import { SdkObject } from '../instrumentation'; import { urlMatches } from '../../utils/isomorphic/urlMatch'; -import { pagePerform, pageExtract } from '../agent/agent'; +import { pageAgentPerform, pageAgentExpect, pageAgentExtract } from '../agent/pageAgent'; import type { Artifact } from '../artifact'; import type { BrowserContext } from '../browserContext'; @@ -322,12 +322,16 @@ export class PageDispatcher extends Dispatcher { - return await pagePerform(progress, this._page, params); + async agentPerform(params: channels.PageAgentPerformParams, progress: Progress): Promise { + return await pageAgentPerform(progress, this._page, params); } - async extract(params: channels.PageExtractParams, progress: Progress): Promise { - const { result, usage } = await pageExtract(progress, this._page, params); + async agentExpect(params: channels.PageAgentExpectParams, progress: Progress): Promise { + await pageAgentExpect(progress, this._page, params); + } + + async agentExtract(params: channels.PageAgentExtractParams, progress: Progress): Promise { + const { result, usage } = await pageAgentExtract(progress, this._page, params); return { result, ...usage }; } diff --git a/packages/playwright-core/src/utils.ts b/packages/playwright-core/src/utils.ts index 78cd6eb6a29d6..2fbe8e3df067b 100644 --- a/packages/playwright-core/src/utils.ts +++ b/packages/playwright-core/src/utils.ts @@ -31,6 +31,7 @@ export * from './utils/isomorphic/stringUtils'; export * from './utils/isomorphic/time'; export * from './utils/isomorphic/timeoutRunner'; export * from './utils/isomorphic/urlMatch'; +export * from './utils/isomorphic/yaml'; export * from './server/utils/ascii'; export * from './server/utils/comparators'; diff --git a/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts b/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts index 27002daec4b21..7cec040fcf159 100644 --- a/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts +++ b/packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts @@ -138,8 +138,9 @@ export const methodMetainfo = new Map { + switch (c) { + case '\\': + return '\\\\'; + case '"': + return '\\"'; + case '\b': + return '\\b'; + case '\f': + return '\\f'; + case '\n': + return '\\n'; + case '\r': + return '\\r'; + case '\t': + return '\\t'; + default: + const code = c.charCodeAt(0); + return '\\x' + code.toString(16).padStart(2, '0'); + } + }) + '"'; +} + +function yamlStringNeedsQuotes(str: string): boolean { + if (str.length === 0) + return true; + + // Strings with leading or trailing whitespace need quotes + if (/^\s|\s$/.test(str)) + return true; + + // Strings containing control characters need quotes + if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/.test(str)) + return true; + + // Strings starting with '-' need quotes + if (/^-/.test(str)) + return true; + + // Strings containing ':' or '\n' followed by a space or at the end need quotes + if (/[\n:](\s|$)/.test(str)) + return true; + + // Strings containing '#' preceded by a space need quotes (comment indicator) + if (/\s#/.test(str)) + return true; + + // Strings that contain line breaks need quotes + if (/[\n\r]/.test(str)) + return true; + + // Strings starting with indicator characters or quotes need quotes + if (/^[&*\],?!>|@"'#%]/.test(str)) + return true; + + // Strings containing special characters that could cause ambiguity + if (/[{}`]/.test(str)) + return true; + + // YAML array starts with [ + if (/^\[/.test(str)) + return true; + + // Non-string types recognized by YAML + if (!isNaN(Number(str)) || ['y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off', 'null'].includes(str.toLowerCase())) + return true; + + return false; +} diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 002616f81a0df..201562bf07c6e 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -1018,24 +1018,6 @@ export interface Page { */ behavior?: 'wait'|'ignoreErrors'|'default' }): Promise; - - /** - * Extract information from the page using the agentic loop, return it in a given Zod format. - * - * **Usage** - * - * ```js - * await page.extract('List of items in the cart', z.object({ - * title: z.string().describe('Item title to extract'), - * price: z.string().describe('Item price to extract'), - * }).array()); - * ``` - * - * @param query Task to perform using agentic loop. - * @param schema - * @param options - */ - extract(query: string, schema: Schema): Promise>; /** * Emitted when the agent makes a turn. */ @@ -3909,45 +3891,6 @@ export interface Page { width?: string|number; }): Promise; - /** - * Perform action using agentic loop. - * - * **Usage** - * - * ```js - * await page.perform('Click submit button'); - * ``` - * - * @param task Task to perform using agentic loop. - * @param options - */ - perform(task: string, options?: { - /** - * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally - * with the `task` as a key. This option allows controlling the cache key explicitly. - */ - key?: string; - - /** - * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. - * Defaults to context-wide value specified in `agent` property. - */ - maxTokens?: number; - - /** - * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. - */ - maxTurns?: number; - }): Promise<{ - usage: { - turns: number; - - inputTokens: number; - - outputTokens: number; - }; - }>; - /** * **NOTE** Use locator-based [locator.press(key[, options])](https://playwright.dev/docs/api/class-locator#locator-press) * instead. Read more about [locators](https://playwright.dev/docs/locators). @@ -5360,6 +5303,8 @@ export interface Page { */ workers(): Array; + agent: PageAgent; + /** * Playwright has ability to mock clock and passage of time. */ @@ -5391,6 +5336,98 @@ export interface Page { [Symbol.asyncDispose](): Promise; } +/** + * + */ +export interface PageAgent { + /** + * Extract information from the page using the agentic loop, return it in a given Zod format. + * + * **Usage** + * + * ```js + * await page.agent.extract('List of items in the cart', z.object({ + * title: z.string().describe('Item title to extract'), + * price: z.string().describe('Item price to extract'), + * }).array()); + * ``` + * + * @param query Task to perform using agentic loop. + * @param schema + * @param options + */ + extract(query: string, schema: Schema): Promise>; + /** + * Expect certain condition to be met. + * + * **Usage** + * + * ```js + * await page.agent.expect('"0 items" to be reported'); + * ``` + * + * @param expectation Expectation to assert. + * @param options + */ + expect(expectation: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + key?: string; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + */ + maxTurns?: number; + }): Promise; + + /** + * Perform action using agentic loop. + * + * **Usage** + * + * ```js + * await page.agent.perform('Click submit button'); + * ``` + * + * @param task Task to perform using agentic loop. + * @param options + */ + perform(task: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + key?: string; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Maximum number of agentic turns during this call, defaults to context-wide value specified in `agent` property. + */ + maxTurns?: number; + }): Promise<{ + usage: { + turns: number; + + inputTokens: number; + + outputTokens: number; + }; + }>; +} + /** * At every point of time, page exposes its current frame tree via the * [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and @@ -22227,8 +22264,7 @@ export interface BrowserContextOptions { acceptDownloads?: boolean; /** - * Agent settings for [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and - * [page.extract(query, schema[, options])](https://playwright.dev/docs/api/class-page#page-extract). + * Agent settings for [page.agent](https://playwright.dev/docs/api/class-page#page-agent). */ agent?: { /** diff --git a/packages/playwright/src/agents/agent.ts b/packages/playwright/src/agents/agent.ts deleted file mode 100644 index c252eafba2f90..0000000000000 --- a/packages/playwright/src/agents/agent.ts +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Loop } from 'playwright-core/lib/mcpBundle'; - -import type z from 'zod'; -import type { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import type { Tool } from '@modelcontextprotocol/sdk/types.js'; -import type * as lowireLoop from '@lowire/loop'; - -type Logger = (category: string, text: string, details?: string) => void; - -export type AgentSpec = { - name: string; - description: string; - model: string; - color: string; - tools: string[]; - instructions: string; - examples: string[]; -}; - -export class Agent> { - readonly loop: lowireLoop.Loop; - readonly spec: AgentSpec; - readonly clients: Map; - readonly resultSchema: Tool['inputSchema']; - - constructor(loopName: 'github' | 'anthropic' | 'openai' | 'google', spec: AgentSpec, clients: Map, resultSchema: Tool['inputSchema']) { - this.loop = new Loop(loopName, { model: spec.model }); - this.spec = spec; - this.clients = clients; - this.resultSchema = resultSchema; - } - - async runTask(task: string, params: any, options?: { logger?: Logger }): Promise> { - const { clients, tools, callTool } = await this._initClients(); - const prompt = this.spec.description; - try { - const { result } = await this.loop.run>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, { - ...options, - tools, - callTool, - resultSchema: this.resultSchema - }); - return result; - } finally { - await this._disconnectFromServers(clients); - } - } - - private async _initClients() { - const clients: Record = {}; - const agentToolNames = new Set(this.spec.tools); - const tools: Tool[] = []; - - for (const [name, client] of this.clients.entries()) { - const list = await client.listTools(); - for (const tool of list.tools) { - if (!agentToolNames.has(name + '/' + tool.name)) - continue; - agentToolNames.delete(name + '/' + tool.name); - tools.push({ ...tool, name: name + '__' + tool.name }); - } - clients[name] = client; - } - - if (agentToolNames.size > 0) - throw new Error(`Required tools not found: ${Array.from(agentToolNames).join(', ')}`); - - const callTool: (params: { name: string, arguments: any}) => Promise = async params => { - const [serverName, toolName] = params.name.split('__'); - const client = clients[serverName]; - if (!client) - throw new Error(`Unknown server: ${serverName}`); - return await client.callTool({ name: toolName, arguments: params.arguments }) as lowireLoop.ToolResult; - }; - return { clients, tools, callTool }; - } - - private async _disconnectFromServers(clients: Record) { - for (const client of Object.values(clients)) - await client.close(); - } -} diff --git a/packages/playwright/src/agents/agentParser.ts b/packages/playwright/src/agents/agentParser.ts index fd3c6bd77da6e..f9523d0ad5b90 100644 --- a/packages/playwright/src/agents/agentParser.ts +++ b/packages/playwright/src/agents/agentParser.ts @@ -17,8 +17,6 @@ import fs from 'fs'; import { yaml } from 'playwright-core/lib/utilsBundle'; -import type { AgentSpec } from './agent'; - type AgentSpecHeader = { name: string; description: string; @@ -27,6 +25,16 @@ type AgentSpecHeader = { tools: string[]; }; +export type AgentSpec = { + name: string; + description: string; + model: string; + color: string; + tools: string[]; + instructions: string; + examples: string[]; +}; + export async function parseAgentSpec(filePath: string): Promise { const source = await fs.promises.readFile(filePath, 'utf-8'); const { header, content } = extractYamlAndContent(source); diff --git a/packages/playwright/src/agents/generateAgents.ts b/packages/playwright/src/agents/generateAgents.ts index a8fc778e9876b..e730654334079 100644 --- a/packages/playwright/src/agents/generateAgents.ts +++ b/packages/playwright/src/agents/generateAgents.ts @@ -24,7 +24,7 @@ import { FullConfigInternal } from '../common/config'; import { defaultSeedFile, findSeedFile, seedFileContent, seedProject } from '../mcp/test/seed'; import { parseAgentSpec } from './agentParser'; -import type { AgentSpec } from './agent'; +import type { AgentSpec } from './agentParser'; /* eslint-disable no-console */ diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index e3ce87e8a3d5e..4f8d26a6cb7f3 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1612,8 +1612,7 @@ interface TestConfig { retries?: number; /** - * Whether to run LLM agent for - * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform): + * Whether to run LLM agent for [page.agent](https://playwright.dev/docs/api/class-page#page-agent): * - "all" disregards existing cache and performs all actions via LLM * - "missing" only performs actions that don't have generated cache actions * - "none" does not talk to LLM at all, relies on the cached actions (default) @@ -2077,8 +2076,7 @@ export interface FullConfig { rootDir: string; /** - * Whether to run LLM agent for - * [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform): + * Whether to run LLM agent for [page.agent](https://playwright.dev/docs/api/class-page#page-agent): * - "all" disregards existing cache and performs all actions via LLM * - "missing" only performs actions that don't have generated cache actions * - "none" does not talk to LLM at all, relies on the cached actions (default) diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 5f0a341247fa8..f1fa2039f8b37 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2135,8 +2135,9 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel { stopCSSCoverage(params?: PageStopCSSCoverageParams, progress?: Progress): Promise; bringToFront(params?: PageBringToFrontParams, progress?: Progress): Promise; updateSubscription(params: PageUpdateSubscriptionParams, progress?: Progress): Promise; - perform(params: PagePerformParams, progress?: Progress): Promise; - extract(params: PageExtractParams, progress?: Progress): Promise; + agentPerform(params: PageAgentPerformParams, progress?: Progress): Promise; + agentExpect(params: PageAgentExpectParams, progress?: Progress): Promise; + agentExtract(params: PageAgentExtractParams, progress?: Progress): Promise; } export type PageAgentTurnEvent = { role: string, @@ -2648,31 +2649,47 @@ export type PageUpdateSubscriptionOptions = { }; export type PageUpdateSubscriptionResult = void; -export type PagePerformParams = { +export type PageAgentPerformParams = { task: string, key?: string, maxTurns?: number, maxTokens?: number, }; -export type PagePerformOptions = { +export type PageAgentPerformOptions = { key?: string, maxTurns?: number, maxTokens?: number, }; -export type PagePerformResult = { +export type PageAgentPerformResult = { turns: number, inputTokens: number, outputTokens: number, }; -export type PageExtractParams = { +export type PageAgentExpectParams = { + expectation: string, + key?: string, + maxTurns?: number, + maxTokens?: number, +}; +export type PageAgentExpectOptions = { + key?: string, + maxTurns?: number, + maxTokens?: number, +}; +export type PageAgentExpectResult = void; +export type PageAgentExtractParams = { query: string, schema: any, + key?: string, maxTurns?: number, + maxTokens?: number, }; -export type PageExtractOptions = { +export type PageAgentExtractOptions = { + key?: string, maxTurns?: number, + maxTokens?: number, }; -export type PageExtractResult = { +export type PageAgentExtractResult = { result: any, turns: number, inputTokens: number, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 7e9d78a1e4817..e0c488da77963 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2022,7 +2022,7 @@ Page: - requestFailed enabled: boolean - perform: + agentPerform: internal: true parameters: task: string @@ -2034,12 +2034,22 @@ Page: inputTokens: int outputTokens: int - extract: + agentExpect: + internal: true + parameters: + expectation: string + key: string? + maxTurns: int? + maxTokens: int? + + agentExtract: internal: true parameters: query: string schema: json + key: string? maxTurns: int? + maxTokens: int? returns: result: json turns: int diff --git a/tests/library/perform-task.spec.ts b/tests/library/perform-task.spec.ts index 906e0061c70a8..de5ce3a8429bc 100644 --- a/tests/library/perform-task.spec.ts +++ b/tests/library/perform-task.spec.ts @@ -34,7 +34,7 @@ test('page.perform', async ({ page, server }) => { // For debugging purposes it is on for now. console.log('agentturn', turn); }); - await page.perform('Fill out the form with the following details:\n' + + await page.agent.perform('Fill out the form with the following details:\n' + 'Name: John Smith\n' + 'Address: 1045 La Avenida St, Mountain View, CA 94043\n' + 'Email: john.smith@at-microsoft.com'); @@ -50,7 +50,7 @@ test('page.perform', async ({ page, server }) => { test('page.perform secret', async ({ page, server }) => { await page.setContent(''); - await page.perform('Enter x-secret-email into the email field'); + await page.agent.perform('Enter x-secret-email into the email field'); await expect(page.locator('body')).toMatchAriaSnapshot(` - textbox "Email Address": secret-email@at-microsoft.com `); @@ -58,8 +58,8 @@ test('page.perform secret', async ({ page, server }) => { test.skip('extract task', async ({ page }) => { await page.goto('https://demo.playwright.dev/todomvc'); - await page.perform('Add "Buy groceries" todo'); - console.log(await page.extract('List todos with their statuses', z.object({ + await page.agent.perform('Add "Buy groceries" todo'); + console.log(await page.agent.extract('List todos with their statuses', z.object({ items: z.object({ title: z.string(), completed: z.boolean() @@ -80,7 +80,7 @@ test('page.perform expect value', async ({ page, server }) => { `); - await page.perform(` + await page.agent.perform(` - Enter "bogus" into the email field - Check that the value is in fact "bogus" - Check that the error message is displayed diff --git a/tests/mcp/browser.eval.ts b/tests/mcp/browser.eval.ts deleted file mode 100644 index 28d528b87ce76..0000000000000 --- a/tests/mcp/browser.eval.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import z from 'zod'; -import { zodToJsonSchema } from 'zod-to-json-schema'; - -import { test, expect } from './fixtures'; - -import type * as lowireLoop from '@lowire/loop'; - -test.use({ mcpHeadless: async ({ headless }, use) => use(headless) }); - -test('fill the form', async ({ loop, client, server }) => { - const { result } = await loop.run<{ price: number, deliveryDays: number }>(` - Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP. - Order a blue table and a green desk. - Use the following details for the order form: - - Name: John Doe - - Address: 123 Main St, Anytown, XYZ state - - Zip Code: 12345 - - Email: john@doe.me - Report back total price for both items, and maximum estimated delivery. - Do not take screenshots. - `, { - tools: (await client.listTools()).tools as lowireLoop.Tool[], - callTool: (params => client.callTool(params)) as lowireLoop.ToolCallback, - resultSchema: zodToJsonSchema(z.object({ - price: z.number().describe('Total price in USD.'), - deliveryDays: z.number().describe('Maximum estimated delivery time in days.'), - })) as lowireLoop.Schema, - }); - expect(result).toEqual({ price: 653.96, deliveryDays: 3 }); -}); diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts index 8860a0251a760..ea5c6d6947d06 100644 --- a/tests/mcp/fixtures.ts +++ b/tests/mcp/fixtures.ts @@ -18,7 +18,6 @@ import fs from 'fs'; import path from 'path'; import { chromium } from 'playwright'; import { Loop } from '@lowire/loop'; -import debug from 'debug'; import { test as baseTest, expect as baseExpect } from '@playwright/test'; import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; @@ -194,31 +193,6 @@ export const test = serverTest.extend { - const cacheFile = path.join(__dirname, '__cache__', 'copilot', sanitizeFileName(test.info().titlePath.join(' ') + '-repeat' + test.info().repeatEachIndex) + '.json'); - const dataBefore = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}'); - let cache = {}; - try { - cache = JSON.parse(dataBefore); - } catch { - } - const loop = new Loop('github', { - model: 'claude-sonnet-4.5', - cache: { - messages: cache, - secrets: { PORT: String(server.PORT) }, - }, - maxTokens: 1_000_000, - debug, - }); - await use(loop); - const dataAfter = JSON.stringify(loop.cache(), null, 2); - if (test.info().status === 'passed' && dataBefore !== dataAfter) { - await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true }); - await fs.promises.writeFile(cacheFile, dataAfter); - } - }, }); async function createTransport(mcpServerType: TestOptions['mcpServerType'], options: { args: string[], env: NodeJS.ProcessEnv, cwd: string }): Promise<{ @@ -348,10 +322,6 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: stri return { client, id }; } -function sanitizeFileName(name: string): string { - return name.replace('.ts', '').replace(/[^a-zA-Z0-9_]+/g, '-'); -} - export const lowireMeta = { 'dev.lowire/history': true, 'dev.lowire/state': true, diff --git a/tests/mcp/playwright.config.ts b/tests/mcp/playwright.config.ts index 65971ac5dab3b..9087f29a485fa 100644 --- a/tests/mcp/playwright.config.ts +++ b/tests/mcp/playwright.config.ts @@ -58,14 +58,5 @@ export default defineConfig({ { name: 'firefox', use: { mcpBrowser: 'firefox' }, metadata: { ...metadata, browserName: 'firefox' }, testDir }, { name: 'webkit', use: { mcpBrowser: 'webkit' }, metadata: { ...metadata, browserName: 'webkit' }, testDir }, ... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' }, metadata: { ...metadata, browserName: 'chromium', channel: 'msedge' }, testDir }] : [], - { - name: 'eval', - testDir, - testMatch: /.*\.eval\.ts/, - repeatEach: 3, // Generate three different trajectories per test. - timeout: process.env.CI ? 30_000 : 180_000, // CI should be cache-only, but local use of LLM is slow. - use: { mcpBrowser: 'chromium' }, - metadata: { ...metadata, browserName: 'chromium' }, - } ], }); diff --git a/utils/generate_types/overrides.d.ts b/utils/generate_types/overrides.d.ts index b50bb7a641949..6013e311712f0 100644 --- a/utils/generate_types/overrides.d.ts +++ b/utils/generate_types/overrides.d.ts @@ -77,7 +77,9 @@ export interface Page { */ behavior?: 'wait'|'ignoreErrors'|'default' }): Promise; +} +export interface PageAgent { extract(query: string, schema: Schema): Promise>; }