feat(coverage): experimental istanbul coverage collection via tracing

docs/src/api/class-tracing.md

@@ -146,6 +146,13 @@ To specify the final trace zip file name, you need to pass `path` option to
 Whether to capture screenshots during tracing. Screenshots are used to build
 a timeline preview.
 
+### option: Tracing.start.coverage
+* since: v1.64
+* langs: js
+- `coverage` <[boolean]>
+
+Whether to collect code coverage from istanbul-instrumented application code. Build the application with an istanbul instrumentation plugin, for example [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right before navigations and page closes, and stores them in istanbul format inside the trace file.
+
 ### option: Tracing.start.snapshots
 * since: v1.12
 * langs: js

docs/src/test-api/class-testoptions.md

@@ -673,6 +673,7 @@ export default defineConfig({
     - `dom` ?<[boolean]> Capture DOM snapshot on every action and record network activity. Optional.
     - `aria` ?<[boolean]> Capture aria snapshot of the page on every action. Optional.
     - `screen` ?<[boolean]> Capture a screenshot of the page on every action. Optional.
+  - `coverage` ?<[boolean]> Whether to collect coverage from istanbul-instrumented application code into the trace. Defaults to false. Optional.
   - `sources` ?<[boolean]> Whether to include source files for trace actions. Defaults to true. Optional.
 
 Whether to record trace for each test. Defaults to `'off'`. The initial run of a test is the "first run"; subsequent runs caused by [retries](../test-retries.md) are "retries".

packages/injected/src/coverageScript.ts

@@ -0,0 +1,165 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { kCoverageStashError, kCoverageStashPrefix } from '@isomorphic/istanbulCoverage';
+
+import type { IstanbulCoverage, IstanbulCoverageDelta, IstanbulFileCoverageDelta } from '@isomorphic/istanbulCoverage';
+
+// Reports `__coverage__` as a delta: reading resets the counters, maps are sent once per file.
+export class CoverageScript {
+  private _global: typeof globalThis;
+  private _takeName: string;
+  private _sessionId: string;
+  private _reportedFiles = new Set<string>();
+  private _onPageHide = () => this._stashCurrent();
+
+  constructor(global: typeof globalThis, takeName: string, sessionId: string) {
+    this._global = global;
+    this._takeName = takeName;
+    this._sessionId = sessionId;
+    (global as any)[takeName] = Object.assign(() => this.take(), { dispose: () => this.dispose() });
+    this._resetCounters();
+    // Counters die with the document and unload-time calls are not delivered.
+    global.addEventListener('pagehide', this._onPageHide);
+  }
+
+  private _resetCounters() {
+    const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__;
+    for (const fileCoverage of Object.values(coverage || {})) {
+      takeCounters(fileCoverage.s);
+      takeCounters(fileCoverage.f);
+      takeBranchCounters(fileCoverage.b);
+    }
+  }
+
+  dispose() {
+    this._global.removeEventListener('pagehide', this._onPageHide);
+    delete (this._global as any)[this._takeName];
+  }
+
+  take(): string[] {
+    const chunks = takeCoverageStashes(this._global, this._sessionId);
+    const delta = this._takeCurrent();
+    if (delta)
+      chunks.push(JSON.stringify({ data: delta }));
+    return chunks;
+  }
+
+  private _takeCurrent(): IstanbulCoverageDelta | undefined {
+    const coverage: IstanbulCoverage | undefined = (this._global as any).__coverage__;
+    if (!coverage)
+      return undefined;
+    const delta: IstanbulCoverageDelta = {};
+    for (const [file, fileCoverage] of Object.entries(coverage)) {
+      const s = takeCounters(fileCoverage.s);
+      const f = takeCounters(fileCoverage.f);
+      const b = takeBranchCounters(fileCoverage.b);
+      // Every file is reported once, so that the never hit ones are accounted for.
+      const isFirstReport = !this._reportedFiles.has(file);
+      if (!isFirstReport && !s && !f && !b)
+        continue;
+      const entry: IstanbulFileCoverageDelta = { path: fileCoverage.path, s: s || {}, f: f || {}, b: b || {} };
+      if (isFirstReport) {
+        entry.statementMap = fileCoverage.statementMap;
+        entry.fnMap = fileCoverage.fnMap;
+        entry.branchMap = fileCoverage.branchMap;
+        this._reportedFiles.add(file);
+      }
+      delta[file] = entry;
+    }
+    return Object.keys(delta).length ? delta : undefined;
+  }
+
+  private _stashCurrent() {
+    const delta = this._takeCurrent();
+    if (!delta)
+      return;
+    // Several documents can pick up the same stash, the id discards the copies.
+    const id = Math.random().toString(36).slice(2);
+    try {
+      this._global.localStorage.setItem(kCoverageStashPrefix + this._sessionId + '.' + id, JSON.stringify({ id, data: delta }));
+    } catch (error) {
+      // The counters are lost, the next take surfaces the error.
+      try {
+        this._global.localStorage.setItem(stashErrorKey(this._sessionId), String(error));
+      } catch {
+      }
+    }
+  }
+}
+
+function stashErrorKey(sessionId: string) {
+  return kCoverageStashPrefix + 'error.' + sessionId;
+}
+
+// Stashes of other sessions are stale, e.g. left by a previous run in a persistent profile.
+export function takeCoverageStashes(global: typeof globalThis, sessionId: string): string[] {
+  const result: string[] = [];
+  let storage: Storage;
+  try {
+    storage = global.localStorage;
+  } catch {
+    return result;
+  }
+  const error = storage.getItem(stashErrorKey(sessionId));
+  if (error) {
+    storage.removeItem(stashErrorKey(sessionId));
+    throw new Error(kCoverageStashError + ': ' + error);
+  }
+  const sessionPrefix = kCoverageStashPrefix + sessionId + '.';
+  const keys: string[] = [];
+  for (let i = 0; i < storage.length; i++) {
+    const key = storage.key(i);
+    if (key && key.startsWith(kCoverageStashPrefix))
+      keys.push(key);
+  }
+  for (const key of keys) {
+    const json = key.startsWith(sessionPrefix) ? storage.getItem(key) : undefined;
+    storage.removeItem(key);
+    if (json)
+      result.push(json);
+  }
+  return result;
+}
+
+function takeCounters(counters: { [key: string]: number }): { [key: string]: number } | undefined {
+  let result: { [key: string]: number } | undefined;
+  for (const key of Object.keys(counters)) {
+    const count = counters[key];
+    if (!count)
+      continue;
+    if (!result)
+      result = {};
+    result[key] = count;
+    counters[key] = 0;
+  }
+  return result;
+}
+
+// Branch counters are positional, so a hit branch is reported with the whole array.
+function takeBranchCounters(counters: { [key: string]: number[] }): { [key: string]: number[] } | undefined {
+  let result: { [key: string]: number[] } | undefined;
+  for (const key of Object.keys(counters)) {
+    const counts = counters[key];
+    if (!counts.some(Boolean))
+      continue;
+    if (!result)
+      result = {};
+    result[key] = counts.slice();
+    counts.fill(0);
+  }
+  return result;
+}

packages/injected/src/storageScript.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { kCoverageStashPrefix } from '@isomorphic/istanbulCoverage';
 import { parseEvaluationResultValue, serializeAsCallArgument, typedArrayToBase64 } from '@isomorphic/utilityScriptSerializers';
 
 import type { IndexedDBDatabase, OPFSEntry, OriginStorage, SetOriginStorage } from '@protocol/structs';
@@ -170,7 +171,9 @@ export class StorageScript {
   }
 
   async collect(record: { indexedDB: boolean, opfs: boolean }): Promise<SerializedStorage> {
-    const localStorage = Object.keys(this._global.localStorage).map(name => ({ name, value: this._global.localStorage.getItem(name)! }));
+    const localStorage = Object.keys(this._global.localStorage)
+        .filter(name => !name.startsWith(kCoverageStashPrefix))
+        .map(name => ({ name, value: this._global.localStorage.getItem(name)! }));
     const result: SerializedStorage = { localStorage };
     if (record.indexedDB) {
       try {

packages/isomorphic/index.ts

@@ -21,6 +21,7 @@ export * from './base64';
 export * from './colors';
 export * from './headers';
 export * from './imageUtils';
+export * from './istanbulCoverage';
 export * from './jsonSchema';
 export * from './locatorGenerators';
 export * from './manualPromise';

packages/isomorphic/istanbulCoverage.ts

@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// The istanbul coverage format, mirrors the shapes from istanbul-lib-coverage.
+
+export type IstanbulLocation = {
+  line: number;
+  column: number;
+};
+
+export type IstanbulRange = {
+  start: IstanbulLocation;
+  end: IstanbulLocation;
+};
+
+export type IstanbulFunctionMapping = {
+  name: string;
+  decl: IstanbulRange;
+  loc: IstanbulRange;
+  line?: number;
+};
+
+export type IstanbulBranchMapping = {
+  type: string;
+  loc: IstanbulRange;
+  locations: IstanbulRange[];
+  line?: number;
+};
+
+export type IstanbulFileCoverage = {
+  path: string;
+  statementMap: { [key: string]: IstanbulRange };
+  fnMap: { [key: string]: IstanbulFunctionMapping };
+  branchMap: { [key: string]: IstanbulBranchMapping };
+  s: { [key: string]: number };
+  f: { [key: string]: number };
+  b: { [key: string]: number[] };
+};
+
+export type IstanbulCoverage = { [file: string]: IstanbulFileCoverage };
+
+// The maps are only sent with the first report of each file.
+export type IstanbulFileCoverageDelta = Partial<IstanbulFileCoverage> & Pick<IstanbulFileCoverage, 'path' | 's' | 'f' | 'b'>;
+
+export type IstanbulCoverageDelta = { [file: string]: IstanbulFileCoverageDelta };
+
+export const kCoverageStashPrefix = '__pwCoverage.';
+export const kCoverageStashError = 'Failed to stash the coverage';
+
+export type IstanbulCoverageChunk = {
+  data: IstanbulCoverageDelta;
+  id?: string;
+};
+
+export function sortedIstanbulCoverage(coverage: Map<string, IstanbulFileCoverage>): IstanbulCoverage {
+  return Object.fromEntries([...coverage.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+export function mergeIstanbulCoverage(into: Map<string, IstanbulFileCoverage>, data: IstanbulCoverageDelta) {
+  for (const [file, fileCov] of Object.entries(data)) {
+    let existing = into.get(file);
+    if (!existing) {
+      existing = { path: fileCov.path, statementMap: {}, fnMap: {}, branchMap: {}, s: {}, f: {}, b: {} };
+      into.set(file, existing);
+    }
+    // The maps can arrive late if the report that carried them was lost.
+    if (fileCov.statementMap && !Object.keys(existing.statementMap).length) {
+      existing.statementMap = fileCov.statementMap;
+      existing.fnMap = fileCov.fnMap || {};
+      existing.branchMap = fileCov.branchMap || {};
+      // Reports only carry the counters that were hit, zero fill the rest.
+      for (const key of Object.keys(existing.statementMap))
+        existing.s[key] = existing.s[key] || 0;
+      for (const key of Object.keys(existing.fnMap))
+        existing.f[key] = existing.f[key] || 0;
+      for (const [key, branch] of Object.entries(existing.branchMap))
+        existing.b[key] = existing.b[key] || branch.locations.map(() => 0);
+    }
+    for (const key of Object.keys(fileCov.s))
+      existing.s[key] = (existing.s[key] || 0) + fileCov.s[key];
+    for (const key of Object.keys(fileCov.f))
+      existing.f[key] = (existing.f[key] || 0) + fileCov.f[key];
+    for (const key of Object.keys(fileCov.b)) {
+      const branches = existing.b[key] || (existing.b[key] = []);
+      fileCov.b[key].forEach((count, i) => branches[i] = (branches[i] || 0) + count);
+    }
+  }
+}

packages/playwright-client/types/types.d.ts

@@ -23268,6 +23268,16 @@ export interface Tracing {
    * @param options
    */
   start(options?: {
+    /**
+     * Whether to collect code coverage from istanbul-instrumented application code. Build the application with an
+     * istanbul instrumentation plugin, for example
+     * [`vite-plugin-istanbul`](https://www.npmjs.com/package/vite-plugin-istanbul) or
+     * [`babel-plugin-istanbul`](https://www.npmjs.com/package/babel-plugin-istanbul), so that pages expose the
+     * `window.__coverage__` object. Playwright collects accumulated counters from all pages and frames, including right
+     * before navigations and page closes, and stores them in istanbul format inside the trace file.
+     */
+    coverage?: boolean;
+
     /**
      * When enabled, the trace is written to an unarchived file that is updated in real time as actions occur, instead of
      * caching changes and archiving them into a zip file at the end. This is useful for live trace viewing during test

packages/playwright-core/src/client/channels.d.ts

@@ -5075,6 +5075,7 @@ export type TracingTracingStartParams = {
   snapshotAria?: boolean,
   snapshotScreen?: boolean,
   screencast?: boolean,
+  coverage?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartOptions = {
@@ -5083,6 +5084,7 @@ export type TracingTracingStartOptions = {
   snapshotAria?: boolean,
   snapshotScreen?: boolean,
   screencast?: boolean,
+  coverage?: boolean,
   live?: boolean,
 };
 export type TracingTracingStartResult = void;

packages/playwright-core/src/client/tracing.ts

@@ -42,7 +42,7 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
     super(parent, type, guid, initializer);
   }
 
-  async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, sources?: boolean, live?: boolean } = {}) {
+  async start(options: { name?: string, title?: string, snapshots?: boolean | { dom?: boolean, aria?: boolean, screen?: boolean }, screenshots?: boolean, coverage?: boolean, sources?: boolean, live?: boolean } = {}) {
     await this._wrapApiCall(async () => {
       this._includeSources = !!options.sources;
       this._isLive = !!options.live;
@@ -53,6 +53,7 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
         snapshotAria: snapshots.aria,
         snapshotScreen: snapshots.screen,
         screencast: options.screenshots,
+        coverage: options.coverage,
         live: options.live,
       }, kNoTimeout);
       const { traceName } = await this._channel.tracingStartChunk({ name: options.name, title: options.title }, kNoTimeout);