From d57491a57d9913bca68c2195a6f488f128fbcb68 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 29 Mar 2023 19:39:42 +0200 Subject: [PATCH 01/10] adding warmup --- resources/benchmark-runner.mjs | 73 +++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index d19fd8662..464270790 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -116,8 +116,61 @@ class PageElement { } } +// The WarmupSuite is used to make sure all runner helper functions and +// classes are compiled, to avoid unnecessary pauses due to delayed +// compilation of runner methods in de middle of the measuring cycle. +const WarmupSuite = { + name: "Warmup", + url: "warmup/index.html", + async prepare(page) { + await page.waitForElement("#testItem"); + }, + tests: [ + // Make sure to run ever page.method once at least + new BenchmarkTestStep(`WarmingUpPageMethods`, (page) => { + let results = []; + results.push(page.querySelector(".testItem")); + results.push(page.querySelectorAll(".item")); + results.push(page.getElementById("testItem")); + }), + new BenchmarkTestStep("WarmingUpPageElementMethods", (page) => { + const item = page.getElementById("testItem"); + console.log("WarmingUpPageElementMethods"); + item.setValue("value"); + item.click(); + item.focus(); + item.dispatchEvent("change"); + item.enter("keypress"); + item.dispatchEvent("input"); + item.enter("keyup"); + }), + new BenchmarkTestStep("WarmingUpPageElementMouseMethods", (page) => { + const item = page.getElementById("testItem"); + const mouseEventOptions = { clientX: 100, clientY: 100, bubbles: true, cancelable: true }; + const wheelEventOptions = { + clientX: 200, + clientY: 200, + deltaMode: 0, + delta: -10, + deltaY: -10, + bubbles: true, + cancelable: true, + }; + console.log("WarmingUpPageElementMouseMethods"); + item.dispatchEvent("mousedown", mouseEventOptions, MouseEvent); + item.dispatchEvent("mousemove", mouseEventOptions, MouseEvent); + item.dispatchEvent("mouseup", mouseEventOptions, MouseEvent); + item.dispatchEvent("wheel", wheelEventOptions, WheelEvent); + }), + ], +} + export class BenchmarkRunner { constructor(suites, client) { + this._reset(suites, client); + } + + _reset(suites, client) { this._suites = suites; this._client = client; this._page = null; @@ -163,6 +216,8 @@ export class BenchmarkRunner { } async runMultipleIterations(iterationCount) { + await this._runWarmupSuite(); + if (this._client?.willStartFirstIteration) await this._client.willStartFirstIteration(iterationCount); for (let i = 0; i < iterationCount; i++) @@ -171,19 +226,33 @@ export class BenchmarkRunner { await this._client.didFinishLastIteration(this._metrics); } + async _runWarmupSuite() { + performance.mark("start-warmup"); + + const savedClient = this._client; + const savedSuites = this._suites; + this._client = undefined; + this._suites = [WarmupSuite]; + + await this._runAllSuites(); + + this._reset(savedSuites, savedClient); + performance.mark("end-warmup"); + performance.measure("warmup", "start-warmup", "end-warmup"); + } + async _runAllSuites() { this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN }; this._removeFrame(); await this._appendFrame(); this._page = new Page(this._frame); - + for (const suite of this._suites) { if (!suite.disabled) await this._runSuite(suite); } - // Remove frame to clear the view for displaying the results. this._removeFrame(); await this._finalize(); From 3d3de6877bdc29d574e46588488265243b03d918 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 29 Mar 2023 19:56:26 +0200 Subject: [PATCH 02/10] suite --- resources/benchmark-runner.mjs | 55 ++++++++++++++-------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 464270790..49a57fe5e 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -167,11 +167,7 @@ const WarmupSuite = { export class BenchmarkRunner { constructor(suites, client) { - this._reset(suites, client); - } - - _reset(suites, client) { - this._suites = suites; + this._suites = [WarmupSuite, ...suites]; this._client = client; this._page = null; this._metrics = { @@ -216,7 +212,6 @@ export class BenchmarkRunner { } async runMultipleIterations(iterationCount) { - await this._runWarmupSuite(); if (this._client?.willStartFirstIteration) await this._client.willStartFirstIteration(iterationCount); @@ -226,28 +221,13 @@ export class BenchmarkRunner { await this._client.didFinishLastIteration(this._metrics); } - async _runWarmupSuite() { - performance.mark("start-warmup"); - - const savedClient = this._client; - const savedSuites = this._suites; - this._client = undefined; - this._suites = [WarmupSuite]; - - await this._runAllSuites(); - - this._reset(savedSuites, savedClient); - performance.mark("end-warmup"); - performance.measure("warmup", "start-warmup", "end-warmup"); - } - async _runAllSuites() { this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN }; this._removeFrame(); await this._appendFrame(); this._page = new Page(this._frame); - + for (const suite of this._suites) { if (!suite.disabled) await this._runSuite(suite); @@ -259,11 +239,19 @@ export class BenchmarkRunner { } async _runSuite(suite) { + const suitePrepareLabel = `suite-${suite.name}-prepare`; + const suiteStartLabel = `suite-${suite.name}-start`; + const suiteEndLabel = `suite-${suite.name}-end`; + + performance.mark(suitePrepareLabel); await this._prepareSuite(suite); - performance.mark(`start-suite-${suite.name}`); + + performance.mark(suiteStartLabel); for (const test of suite.tests) await this._runTestAndRecordResults(suite, test); - performance.mark(`end-suite-${suite.name}`); + performance.mark(suiteEndLabel); + + performance.measure(`suite-${suite.name}`, suitePrepareLabel, suiteEndLabel); } async _prepareSuite(suite) { @@ -324,14 +312,17 @@ export class BenchmarkRunner { } async _recordTestResults(suite, test, syncTime, asyncTime, unused_height, testDoneCallback) { - const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 }; - const total = syncTime + asyncTime; - this._measuredValues.tests[suite.name] = suiteResults; - suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; - suiteResults.total += total; - - if (this._client?.didRunTest) - await this._client.didRunTest(suite, test); + // Skip reporting updates for the warmup suite. + if (suite !== WarmupSuite) { + const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 }; + const total = syncTime + asyncTime; + this._measuredValues.tests[suite.name] = suiteResults; + suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; + suiteResults.total += total; + + if (this._client?.didRunTest) + await this._client.didRunTest(suite, test); + } testDoneCallback(); } From be6f93b44ea362466256e44f293f8eb88baa5ae5 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 29 Mar 2023 19:56:35 +0200 Subject: [PATCH 03/10] adding warmup page --- resources/warmup/index.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 resources/warmup/index.html diff --git a/resources/warmup/index.html b/resources/warmup/index.html new file mode 100644 index 000000000..fc2da7a89 --- /dev/null +++ b/resources/warmup/index.html @@ -0,0 +1,18 @@ + + + + + Warmup Page + + +

WARMUP

+
+
item 1
+
item 2
+
item 3
+
item 4
+
item 5
+
item 6
+
+ + From 2a98cb751f619f54d2a1e59a1b654509964438aa Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 29 Mar 2023 19:57:00 +0200 Subject: [PATCH 04/10] more runner --- resources/benchmark-runner.mjs | 6 +++--- resources/warmup/index.html | 30 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 49a57fe5e..4a4ec578b 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -127,7 +127,7 @@ const WarmupSuite = { }, tests: [ // Make sure to run ever page.method once at least - new BenchmarkTestStep(`WarmingUpPageMethods`, (page) => { + new BenchmarkTestStep("WarmingUpPageMethods", (page) => { let results = []; results.push(page.querySelector(".testItem")); results.push(page.querySelectorAll(".item")); @@ -163,7 +163,7 @@ const WarmupSuite = { item.dispatchEvent("wheel", wheelEventOptions, WheelEvent); }), ], -} +}; export class BenchmarkRunner { constructor(suites, client) { @@ -212,7 +212,6 @@ export class BenchmarkRunner { } async runMultipleIterations(iterationCount) { - if (this._client?.willStartFirstIteration) await this._client.willStartFirstIteration(iterationCount); for (let i = 0; i < iterationCount; i++) @@ -233,6 +232,7 @@ export class BenchmarkRunner { await this._runSuite(suite); } + // Remove frame to clear the view for displaying the results. this._removeFrame(); await this._finalize(); diff --git a/resources/warmup/index.html b/resources/warmup/index.html index fc2da7a89..80f84178b 100644 --- a/resources/warmup/index.html +++ b/resources/warmup/index.html @@ -1,18 +1,18 @@ - - - Warmup Page - - -

WARMUP

-
-
item 1
-
item 2
-
item 3
-
item 4
-
item 5
-
item 6
-
- + + + Warmup Page + + +

WARMUP

+
+
item 1
+
item 2
+
item 3
+
item 4
+
item 5
+
item 6
+
+ From d1dab13e726b9f9695d658cbaf5931094afc3024 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 11 Apr 2023 16:40:48 +0200 Subject: [PATCH 05/10] pre-format --- resources/params.mjs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/resources/params.mjs b/resources/params.mjs index af6c779e9..4827dd38e 100644 --- a/resources/params.mjs +++ b/resources/params.mjs @@ -7,6 +7,7 @@ class Params { startAutomatically = false; iterationCount = 10; suites = []; + useWarmupSuite = false; constructor(searchParams = undefined) { if (searchParams) @@ -48,6 +49,11 @@ class Params { const unused = Array.from(searchParams.keys()); if (unused.length > 0) console.error("Got unused search params", unused); + + if (searchParams.has("useWarmupSuite")) { + this.useWarmupSuite = true; + searchParams.delete("useWarmupSuite"); + } } toSearchParams() { From 740915dbd85378a3c348c68bf4ddcc40b6fada35 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 11 Apr 2023 16:41:15 +0200 Subject: [PATCH 06/10] formatting --- resources/params.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/params.mjs b/resources/params.mjs index 4827dd38e..68ffb4f28 100644 --- a/resources/params.mjs +++ b/resources/params.mjs @@ -49,7 +49,7 @@ class Params { const unused = Array.from(searchParams.keys()); if (unused.length > 0) console.error("Got unused search params", unused); - + if (searchParams.has("useWarmupSuite")) { this.useWarmupSuite = true; searchParams.delete("useWarmupSuite"); From 53bb3511854624428f450384b7e54c902bf67db1 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 11 Apr 2023 17:29:10 +0200 Subject: [PATCH 07/10] adding params --- resources/benchmark-runner.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index ea96721b1..a91e272af 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -167,7 +167,9 @@ const WarmupSuite = { export class BenchmarkRunner { constructor(suites, client) { - this._suites = [WarmupSuite, ...suites]; + this._suites = suites; + if (params.useWarmupSuite) + this._suites = [WarmupSuite, ...suites]; this._client = client; this._page = null; this._metrics = { From 27dfc5a8005bd1cf5dfb26561219ee413b1abcd5 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 11 Apr 2023 17:45:49 +0200 Subject: [PATCH 08/10] fix tests --- tests/benchmark-runner-tests.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/benchmark-runner-tests.mjs b/tests/benchmark-runner-tests.mjs index bc0906e8e..335c6b414 100644 --- a/tests/benchmark-runner-tests.mjs +++ b/tests/benchmark-runner-tests.mjs @@ -167,9 +167,10 @@ describe("BenchmarkRunner", () => { it("should run and record results for every test in suite", async () => { assert.calledThrice(_runTestAndRecordResultsStub); - assert.calledWith(peformanceMarkSpy, "start-suite-Suite 1"); - assert.calledWith(peformanceMarkSpy, "end-suite-Suite 1"); - expect(peformanceMarkSpy.callCount).to.equal(2); + assert.calledWith(peformanceMarkSpy, "suite-Suite 1-prepare"); + assert.calledWith(peformanceMarkSpy, "suite-Suite 1-start"); + assert.calledWith(peformanceMarkSpy, "suite-Suite 1-end"); + expect(peformanceMarkSpy.callCount).to.equal(3); }); }); }); From 50bf1496843fa42cf109d897dfd492eb7dca076a Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 15 May 2023 23:11:10 +0200 Subject: [PATCH 09/10] pre-format --- resources/benchmark-runner.mjs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 64b78985b..daf890dd2 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -118,7 +118,7 @@ class PageElement { // The WarmupSuite is used to make sure all runner helper functions and // classes are compiled, to avoid unnecessary pauses due to delayed -// compilation of runner methods in de middle of the measuring cycle. +// compilation of runner methods in the middle of the measuring cycle. const WarmupSuite = { name: "Warmup", url: "warmup/index.html", @@ -135,7 +135,6 @@ const WarmupSuite = { }), new BenchmarkTestStep("WarmingUpPageElementMethods", (page) => { const item = page.getElementById("testItem"); - console.log("WarmingUpPageElementMethods"); item.setValue("value"); item.click(); item.focus(); @@ -156,7 +155,6 @@ const WarmupSuite = { bubbles: true, cancelable: true, }; - console.log("WarmingUpPageElementMouseMethods"); item.dispatchEvent("mousedown", mouseEventOptions, MouseEvent); item.dispatchEvent("mousemove", mouseEventOptions, MouseEvent); item.dispatchEvent("mouseup", mouseEventOptions, MouseEvent); From 1a638fd7f2571168bb3e4767b96def2334474ff2 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 15 May 2023 23:39:53 +0200 Subject: [PATCH 10/10] pre-format --- resources/benchmark-runner.mjs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index daf890dd2..7b236fe3f 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -313,16 +313,18 @@ export class BenchmarkRunner { async _recordTestResults(suite, test, syncTime, asyncTime, unused_height, testDoneCallback) { // Skip reporting updates for the warmup suite. - if (suite !== WarmupSuite) { - const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 }; - const total = syncTime + asyncTime; - this._measuredValues.tests[suite.name] = suiteResults; - suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; - suiteResults.total += total; - - if (this._client?.didRunTest) - await this._client.didRunTest(suite, test); + if (suite === WarmupSuite) { + testDoneCallback(); + return; } + const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 }; + const total = syncTime + asyncTime; + this._measuredValues.tests[suite.name] = suiteResults; + suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total }; + suiteResults.total += total; + + if (this._client?.didRunTest) + await this._client.didRunTest(suite, test); testDoneCallback(); }