diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 5c2b7521971d8..55399a4d678f8 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache. - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. +### option: Page.agent.expect +* since: v1.58 +- `expect` <[Object]> + - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms. + ### option: Page.agent.limits * since: v1.58 - `limits` <[Object]> diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md index 26242e8c4428d..0f04e2b04d851 100644 --- a/docs/src/api/class-pageagent.md +++ b/docs/src/api/class-pageagent.md @@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported'); Expectation to assert. +### option: PageAgent.expect.timeout +* since: v1.58 +- `timeout` <[float]> + +Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout. + ### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -68,6 +74,13 @@ Task to perform using agentic loop. * since: v1.58 - `schema` <[z.ZodSchema]> +### option: PageAgent.extract.timeout +* since: v1.58 +- `timeout` <[float]> + +Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -94,6 +107,13 @@ await agent.perform('Click submit button'); Task to perform using agentic loop. +### option: PageAgent.perform.timeout +* since: v1.58 +- `timeout` <[float]> + +Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 9b7bf8f3d4af4..69257b93db23d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property. -## page-agent-timeout -* since: v1.58 -- `timeout` <[int]> - -Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. - ## page-agent-call-options-v1.58 - %%-page-agent-cache-key-%% - %%-page-agent-max-tokens-%% - %%-page-agent-max-actions-%% - %%-page-agent-max-action-retries-%% -- %%-page-agent-timeout-%% ## fetch-param-url - `url` <[string]> diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts index ce18bf46aa252..2be0082e4d79a 100644 --- a/packages/injected/src/injectedScript.ts +++ b/packages/injected/src/injectedScript.ts @@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript'; export type FrameExpectParams = Omit & { expectedValue?: any; timeoutForLogs?: number; - explicitTimeout?: number; - noPreChecks?: boolean; + noAutoWaiting?: boolean; }; export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable'; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index e7c027f2a8941..adc7654de9599 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -865,7 +865,9 @@ export class Page extends ChannelOwner implements api.Page systemPrompt: options.systemPrompt, }; const { agent } = await this._channel.agent(params); - return PageAgent.from(agent); + const pageAgent = PageAgent.from(agent); + pageAgent._expectTimeout = options?.expect?.timeout; + return pageAgent; } async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> { diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts index 2837f5bbb11f3..ec6a16cc9aeab 100644 --- a/packages/playwright-core/src/client/pageAgent.ts +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -22,14 +22,9 @@ import { Page } from './page'; import type * as api from '../../types/types'; import type * as channels from '@protocol/channels'; -type PageAgentOptions = { - maxTokens?: number; - maxTurns?: number; - cacheKey?: string; -}; - export class PageAgent extends ChannelOwner implements api.PageAgent { private _page: Page; + _expectTimeout?: number; static from(channel: channels.PageAgentChannel): PageAgent { return (channel as any)._object; @@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner implement this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params)); } - async expect(expectation: string, options: PageAgentOptions = {}) { - await this._channel.expect({ expectation, ...options }); + async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) { + const timeout = options.timeout ?? this._expectTimeout ?? 5000; + await this._channel.expect({ expectation, ...options, timeout }); } - async perform(task: string, options: PageAgentOptions = {}) { - const { usage } = await this._channel.perform({ task, ...options }); + async perform(task: string, options: channels.PageAgentPerformOptions = {}) { + const timeout = this._page._timeoutSettings.timeout(options); + const { usage } = await this._channel.perform({ task, ...options, timeout }); return { usage }; } - async extract(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { - const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); + async extract(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { + const timeout = this._page._timeoutSettings.timeout(options); + const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout }); return { result, usage }; } diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts index 9f25d3fec282e..9773c29b55a03 100644 --- a/packages/playwright-core/src/server/agent/actionRunner.ts +++ b/packages/playwright-core/src/server/agent/actionRunner.ts @@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page } async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) { - // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress. - const timeout = expectTimeout(mode); const result = await frame.expect(progress, selector, { ...options, - timeoutForLogs: timeout, - explicitTimeout: timeout, - // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway. - noPreChecks: mode === 'generate', + // When generating, we want the expect to pass or fail immediately and give feedback to the model. + noAutoWaiting: mode === 'generate', + timeoutForLogs: mode === 'generate' ? undefined : progress.timeout, }); if (!result.matches === !options.isNot) { const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received; @@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r expectation, locator: selector ? asLocatorDescription('javascript', selector) : undefined, timedOut: result.timedOut, - timeout, + timeout: mode === 'generate' ? undefined : progress.timeout, printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`, printedReceived: result.errorMessage ? '' : `Received: ${received}`, errorMessage: result.errorMessage, @@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.value', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Value', params }; } else if (action.type === 'checkbox' || action.type === 'radio') { @@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.checked', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Checked', params }; } else { @@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.visible', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Visible', params }; } @@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.match.snapshot', expectedText: [], isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params }; } @@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.url', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect URL', params }; } @@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.title', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Title', params }; } @@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions }; return callMetadata; } - -function expectTimeout(mode: 'generate' | 'run') { - return mode === 'generate' ? 0 : 5000; -} diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts index 120f611390f5c..7b22731aa58e0 100644 --- a/packages/playwright-core/src/server/agent/pageAgent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use ### Task ${userTask} `; - + progress.disableTimeout(); await runLoop(progress, context, performTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe ### Expectation ${expectation} `; - + progress.disableTimeout(); await runLoop(progress, context, expectTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -101,7 +101,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To const apiCacheTextBefore = context.agentParams.apiCacheFile ? await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}'; - const apiCacheBefore = JSON.parse(apiCacheTextBefore); + const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}'); const loop = new Loop({ api: context.agentParams.api as any, diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 5431ea6a40e26..a09b41d050cff 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1389,14 +1389,10 @@ export class Frame extends SdkObject { progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } }; }; try { - // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort. - const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined; - timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ }); - // Step 1: perform locator handlers checkpoint with a specified timeout. if (selector) progress.log(`waiting for ${this._asLocator(selector)}`); - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); // Step 2: perform one-shot expect check without a timeout. @@ -1404,19 +1400,19 @@ export class Frame extends SdkObject { // that should succeed when the locator is already visible. try { const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true); - if (resultOneShot.matches !== options.isNot) + if (options.noAutoWaiting || resultOneShot.matches !== options.isNot) return resultOneShot; } catch (e) { - if (this.isNonRetriableError(e)) + if (options.noAutoWaiting || this.isNonRetriableError(e)) throw e; // Ignore any other errors from one-shot, we'll handle them during retries. } // Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time. const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => { - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); - const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise); + const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false); if (matches === options.isNot) { // Keep waiting in these cases: // expect(locator).conditionThatDoesNotMatch @@ -1446,9 +1442,9 @@ export class Frame extends SdkObject { } } - private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise) { + private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) { // The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted. - const race = (p: Promise) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p)); + const race = (p: Promise) => noAbort ? p : progress.race(p); const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined; const { frame, info } = selectorInFrame || { frame: this, info: undefined }; diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts index b6cb8201fb27d..07b580bff6376 100644 --- a/packages/playwright-core/src/server/progress.ts +++ b/packages/playwright-core/src/server/progress.ts @@ -61,10 +61,14 @@ export class ProgressController { const deadline = timeout ? monotonicTime() + timeout : 0; assert(this._state === 'before'); this._state = 'running'; + let timer: NodeJS.Timeout | undefined; const progress: Progress = { timeout: timeout ?? 0, deadline, + disableTimeout: () => { + clearTimeout(timer); + }, log: message => { if (this._state === 'running') this.metadata.log.push(message); @@ -87,10 +91,10 @@ export class ProgressController { signal: this._controller.signal, }; - let timer: NodeJS.Timeout | undefined; if (deadline) { const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`); timer = setTimeout(() => { + // TODO: migrate this to "progress.disableTimeout()". if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime) return; if (this._state === 'running') { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 82b978fb8caf2..874ba50ef2d00 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -477,6 +477,9 @@ const playwrightFixtures: Fixtures = ({ limits: agentOptions?.limits, secrets: agentOptions?.secrets, systemPrompt: agentOptions?.systemPrompt, + expect: { + timeout: testInfoImpl._projectInternal.expect?.timeout, + }, }); await use(agent); diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts index 8e389e0c31ea4..f7abd096ac5b9 100644 --- a/packages/protocol/src/progress.d.ts +++ b/packages/protocol/src/progress.d.ts @@ -37,6 +37,7 @@ import type { CallMetadata } from './callMetadata'; export interface Progress { timeout: number; deadline: number; + disableTimeout(): void; log(message: string): void; race(promise: Promise | Promise[]): Promise; wait(timeout: number): Promise; // timeout = 0 here means "wait 0 ms", not forever. diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json new file mode 100644 index 0000000000000..1109e7715d4e2 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json @@ -0,0 +1,30 @@ +{ + "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll verify that the submit button is visible on the page." + }, + { + "type": "tool_call", + "name": "browser_expect_visible", + "arguments": { + "role": "button", + "accessibleName": "Submit", + "_is_done": true + }, + "id": "toolu_012RgUS8buCjrQRgt4w2sYFf" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2203, + "output": 107 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json index bb23b43b9d30c..0f0ff37ab65b6 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition." + "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "My Page Title", "_is_done": true }, - "id": "toolu_01VvWdf4VVFNQuDaEtTsAu3K" + "id": "toolu_01R1QyAwfxk2KSz3tsAx9RCA" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2195, - "output": 106 + "output": 105 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json index e14e18a7d207d..a4ae141362c51 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page title is \"Other Title\" using the browser_expect_title tool." + "text": "I need to verify that the page title is \"Other Title\" using one of the browser_expect_* tools." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "Other Title", "_is_done": true }, - "id": "toolu_017UT5LEe8JRoB53R1YGKrYD" + "id": "toolu_017ZvXqNbcab9Uyjzw3VuByK" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2194, - "output": 97 + "output": 100 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json index c3f3b6c22520b..56fadb43dc1e4 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." + "text": "I'll verify that the page URL is /counter.html." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "url": "/counter.html", "_is_done": true }, - "id": "toolu_017jkU3FcrQpAVJgtjrzYovK" + "id": "toolu_01PkjsZaM5eEGtjLtMixyJUk" } ], "stopReason": { @@ -23,25 +23,25 @@ }, "usage": { "input": 2240, - "output": 99 + "output": 91 } }, - "f6d9bc80fd5bad48f6a886d7ec77c8d5889521b1": { + "71a7a4f51b3994a200f239eda9b505454825a79f": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I need to use a regex pattern to match the URL since it includes a server prefix. Let me update the expectation to match the actual URL pattern." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/counter\\.html$/", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_01JQuC1CJR7W6LiwAnvjaiDX" + "id": "toolu_01Wwq7Pbe2DthqpmskgGfCzk" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2441, - "output": 109 + "input": 2433, + "output": 111 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json index f034545e132aa..3f76a92de99db 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page URL matches the `/counter` pattern using the browser_expect_url tool with a regex pattern." + "text": "I'll verify that the page URL matches the /counter pattern using the browser_expect_url tool with a regex pattern." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_01J38E1DkiNA7JsvQtGx3otW" + "id": "toolu_01JxsK661QmxpScYeTMCfWTZ" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2239, - "output": 103 + "output": 102 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json index 19a6a08b333e3..c8ae66692ec94 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json @@ -14,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_01Qen21mVwEVb2JsEVavTWbf" + "id": "toolu_01YVkbby1j3FBXk2o5Bi4cjP" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json index 43b309b180056..b88a2b2d35cfc 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json @@ -14,7 +14,7 @@ "url": "/counter.html", "_is_done": true }, - "id": "toolu_01B3LSMGJoDrDfCZKwa3neZ9" + "id": "toolu_01BrgLhGzD2dYWeu75KGgxKv" } ], "stopReason": { @@ -26,22 +26,22 @@ "output": 99 } }, - "99f45f7c71e64f612bfc0bcb148faf235ccfe006": { + "8315bc81a0dfc915cb8be72ad00102f627e370dd": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue. The actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I see the issue. The actual URL includes the server portion. Let me use a regex pattern to match the URL ending with /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/counter\\.html$/", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_01Ufux7hKwV9Djp9jT1JgYwJ" + "id": "toolu_01KtWVz2EdJjx2V9viYeqs71" } ], "stopReason": { @@ -50,7 +50,7 @@ }, "usage": { "input": 2441, - "output": 109 + "output": 110 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json new file mode 100644 index 0000000000000..1cd1ae76339cf --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json @@ -0,0 +1,30 @@ +{ + "554366043f6477d30a36655c10cbb4ec0eaa21d4": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll click the Fox button for you." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Fox button", + "ref": "e3", + "_is_done": true + }, + "id": "toolu_01TBMe2oPqGUpNuPnFnDeZJe" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3066, + "output": 101 + } + } +} \ No newline at end of file diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts index c7befd887ab86..f301a6c5ab1b9 100644 --- a/tests/library/agent-expect.spec.ts +++ b/tests/library/agent-expect.spec.ts @@ -196,16 +196,36 @@ test('expect timeout during run', async ({ context }) => { { const { page, agent } = await runAgent(context); await page.setContent(``); - const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e); + const error = await agent.expect('submit button is visible', { timeout: 3000 }).catch(e => e); expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed Locator: getByRole('button', { name: 'Submit' }) Expected: visible -Timeout: 5000ms +Timeout: 3000ms Error: element(s) not found Call log: - - Expect Visible with timeout 5000ms`); + - Expect Visible with timeout 3000ms`); + } +}); + +test('expect timeout during run from agent options', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(``); + await agent.expect('submit button is visible'); + } + expect(await cacheObject()).toEqual({ + 'submit button is visible': { + actions: [expect.objectContaining({ method: 'expectVisible' })], + }, + }); + { + const { page, agent } = await runAgent(context, { expect: { timeout: 3000 } }); + await page.setContent(``); + const error = await agent.expect('submit button is visible').catch(e => e); + expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed`); + expect(stripAnsi(error.message)).toContain(`Expect Visible with timeout 3000ms`); } }); diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts index 88492bca9607d..5a718082df48d 100644 --- a/tests/library/agent-helpers.ts +++ b/tests/library/agent-helpers.ts @@ -55,9 +55,10 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio return { page, agent }; } -export async function runAgent(context: BrowserContext, options: { secrets?: Record } = {}) { +export async function runAgent(context: BrowserContext, options: AgentOptions = {}) { const page = await context.newPage(); const agent = await page.agent({ + ...options, cache: { cacheFile: cacheFile() }, ...{ _doNotRenderActive: true }, }); diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts index c7f80dd3ec9f0..9a14b409a20df 100644 --- a/tests/library/agent-perform.spec.ts +++ b/tests/library/agent-perform.spec.ts @@ -174,6 +174,28 @@ test('perform run timeout', async ({ context }) => { } }); +test('perform run timeout inherited from page', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(` + + + `); + await agent.perform('click the Fox button'); + } + { + const { page, agent } = await runAgent(context); + await page.setContent(` + + + `); + page.setDefaultTimeout(3000); + const error = await agent.perform('click the Fox button').catch(e => e); + expect(error.message).toContain('Timeout 3000ms exceeded.'); + expect(error.message).toContain(`waiting for getByRole('button', { name: 'Fox' })`); + } +}); + test('invalid cache file throws error', async ({ context }) => { await setCacheObject({ 'some key': {