chore(tracing): reference trace and har blobs via relative file paths

packages/isomorphic/trace/entries.ts

@@ -48,7 +48,7 @@ export type ContextEntry = {
 export type PageEntry = {
   pageId: string,
   screencastFrames: {
-    sha1: string,
+    file: string,
     timestamp: number,
     frameSwapWallTime?: number,
     width: number,

packages/isomorphic/trace/snapshotRenderer.ts

@@ -71,7 +71,7 @@ export class SnapshotRenderer {
     const closestFrame = (wallTime && this._screencastFrames[0]?.frameSwapWallTime)
       ? findClosest(this._screencastFrames, frame => frame.frameSwapWallTime!, wallTime)
       : findClosest(this._screencastFrames, frame => frame.timestamp, timestamp);
-    return closestFrame?.sha1;
+    return closestFrame?.file;
   }
 
   render(): RenderedFrameSnapshot {
@@ -234,14 +234,14 @@ export class SnapshotRenderer {
         if (index >= 0 && index < this._snapshots.length)
           override = this._snapshots[index].resourceOverrides.find(o => o.url === url);
       }
-      if (override?.sha1) {
+      if (override?.file) {
         result = {
           ...result,
           response: {
             ...result.response,
             content: {
               ...result.response.content,
-              _sha1: override.sha1,
+              _file: override.file,
             }
           },
         };

packages/isomorphic/trace/snapshotServer.ts

@@ -21,10 +21,10 @@ import type { ResourceSnapshot } from '@trace/snapshot';
 
 export class SnapshotServer {
   private _snapshotStorage: SnapshotStorage;
-  private _resourceLoader: (sha1: string) => Promise<Blob | undefined>;
+  private _resourceLoader: (file: string) => Promise<Blob | undefined>;
   private _snapshotIds = new Map<string, SnapshotRenderer>();
 
-  constructor(snapshotStorage: SnapshotStorage, resourceLoader: (sha1: string) => Promise<Blob | undefined>) {
+  constructor(snapshotStorage: SnapshotStorage, resourceLoader: (file: string) => Promise<Blob | undefined>) {
     this._snapshotStorage = snapshotStorage;
     this._resourceLoader = resourceLoader;
   }
@@ -41,10 +41,10 @@ export class SnapshotServer {
 
   async serveClosestScreenshot(pageOrFrameId: string, searchParams: URLSearchParams): Promise<Response> {
     const snapshot = this._snapshot(pageOrFrameId, searchParams);
-    const sha1 = snapshot?.closestScreenshot();
-    if (!sha1)
+    const file = snapshot?.closestScreenshot();
+    if (!file)
       return new Response(null, { status: 404 });
-    return new Response(await this._resourceLoader(sha1));
+    return new Response(await this._resourceLoader(file));
   }
 
   serveSnapshotInfo(pageOrFrameId: string, searchParams: URLSearchParams): Response {
@@ -85,8 +85,8 @@ export class SnapshotServer {
     if (!resource)
       return new Response(null, { status: 404 });
 
-    const sha1 = resource.response.content._sha1;
-    const content = sha1 ? await this._resourceLoader(sha1) || new Blob([]) : new Blob([]);
+    const file = resource.response.content._file;
+    const content = file ? await this._resourceLoader(file) || new Blob([]) : new Blob([]);
 
     let contentType = resource.response.content.mimeType;
     const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType);

packages/isomorphic/trace/traceLoader.ts

@@ -48,7 +48,7 @@ export class TraceLoader {
       const match = entryName.match(/(.+)\.trace$/);
       if (match && (!prefix || prefix  === match[1]))
         prefixes.push(match[1] || '');
-      if (entryName.includes('src@'))
+      if (entryName.startsWith('src/') || entryName.includes('src@'))
         hasSource = true;
     }
     if (!prefixes.length)
@@ -97,10 +97,10 @@ export class TraceLoader {
       unzipProgress?.(++done, total);
 
       for (const resource of contextEntry.resources) {
-        if (resource.request.postData?._sha1)
-          this._resourceToContentType.set(resource.request.postData._sha1, stripEncodingFromContentType(resource.request.postData.mimeType));
-        if (resource.response.content?._sha1)
-          this._resourceToContentType.set(resource.response.content._sha1, stripEncodingFromContentType(resource.response.content.mimeType));
+        if (resource.request.postData?._file)
+          this._resourceToContentType.set(resource.request.postData._file, stripEncodingFromContentType(resource.request.postData.mimeType));
+        if (resource.response.content?._file)
+          this._resourceToContentType.set(resource.response.content._file, stripEncodingFromContentType(resource.response.content.mimeType));
       }
 
       this.contextEntries.push(contextEntry);
@@ -113,9 +113,9 @@ export class TraceLoader {
     return this._backend.hasEntry(filename);
   }
 
-  async resourceForSha1(sha1: string): Promise<Blob | undefined> {
-    const blob = await this._backend.readBlob('resources/' + sha1);
-    const contentType = this._resourceToContentType.get(sha1);
+  async resourceEntry(file: string): Promise<Blob | undefined> {
+    const blob = await this._backend.readBlob(file);
+    const contentType = this._resourceToContentType.get(file);
     // "x-unknown" in the har means "no content type".
     if (!blob || contentType === undefined || contentType === 'x-unknown')
       return blob;

packages/isomorphic/trace/traceModernizer.ts

@@ -200,9 +200,48 @@ export class TraceModernizer {
     let events = [event];
     for (; version < latestVersion; ++version)
       events = (this as any)[`_modernize_${version}_to_${version + 1}`].call(this, events);
+    for (const e of events)
+      this._normalizeResourceReferences(e);
     return events;
   }
 
+  // Traces recorded before trace-relative paths referenced blobs by bare sha1-style names:
+  // `_sha1` in har entry content, `sha1` in snapshot resource overrides, screencast frames
+  // and attachments.
+  private _normalizeResourceReferences(event: any) {
+    if (event.type === 'resource-snapshot') {
+      const { request, response } = event.snapshot;
+      if (request?.postData?._sha1) {
+        request.postData._file = 'resources/' + request.postData._sha1;
+        delete request.postData._sha1;
+      }
+      if (response?.content?._sha1) {
+        response.content._file = 'resources/' + response.content._sha1;
+        delete response.content._sha1;
+      }
+    }
+    if (event.type === 'frame-snapshot') {
+      for (const override of event.snapshot.resourceOverrides || []) {
+        if (override.sha1) {
+          override.file = 'resources/' + override.sha1;
+          delete override.sha1;
+        }
+      }
+    }
+    if (event.type === 'screencast-frame' && event.sha1) {
+      event.file = 'resources/' + event.sha1;
+      delete event.sha1;
+    }
+    if (event.type === 'after' || event.type === 'action') {
+      for (const attachment of event.attachments || []) {
+        if (attachment.sha1) {
+          attachment.file = 'resources/' + attachment.sha1;
+          delete attachment.sha1;
+        }
+      }
+    }
+  }
+
   _modernize_0_to_1(events: any[]): any[] {
     for (const event of events) {
       if (event.type !== 'action')

packages/playwright-core/src/server/har/harRecorder.ts

@@ -33,6 +33,7 @@ export class HarRecorder implements HarTracerDelegate {
   private _fs = new SerializedFS();
   private _harFilePath: string;
   private _resourcesDir: string;
+  private _relativeResourcesDir: string;
   private _isFlushed: boolean = false;
   private _tracer: HarTracer;
   private _entries: har.Entry[] = [];
@@ -42,18 +43,23 @@ export class HarRecorder implements HarTracerDelegate {
     this._context = context;
     const isServer = !!context.attribution.playwright.options.isServer;
     this._harFilePath = !isServer && options.harPath ? options.harPath : path.join(fallbackDir, `${harId}.har`);
-    if (!isServer && options.resourcesDir)
+    const harFileDir = path.dirname(this._harFilePath);
+    if (!isServer && options.resourcesDir) {
       this._resourcesDir = options.resourcesDir;
-    else if (!isServer && options.harPath)
-      this._resourcesDir = path.dirname(options.harPath);
-    else
+      this._relativeResourcesDir = path.relative(harFileDir, this._resourcesDir).split(path.sep).join('/');
+    } else if (!isServer && options.harPath) {
+      this._resourcesDir = harFileDir;
+      this._relativeResourcesDir = '';
+    } else {
+      // Staging layout for the zip archive, where resources end up next to har.har.
       this._resourcesDir = path.join(fallbackDir, `${harId}-resources`);
+      this._relativeResourcesDir = '';
+    }
     const urlFilterRe = options.urlRegexSource !== undefined && options.urlRegexFlags !== undefined ? new RegExp(options.urlRegexSource, options.urlRegexFlags) : undefined;
     const content = options.content || 'embed';
     this._tracer = new HarTracer(context, page, this, {
       content,
       slimMode: options.mode === 'minimal',
-      includeTraceInfo: false,
       recordRequestOverrides: true,
       waitForContentOnStop: true,
       urlFilter: urlFilterRe ?? options.urlGlob,
@@ -69,20 +75,28 @@ export class HarRecorder implements HarTracerDelegate {
   onEntryFinished(entry: har.Entry) {
   }
 
-  onContentBlob(sha1: string, buffer: Buffer) {
-    if (this._writtenContentEntries.has(sha1))
-      return;
+  onContentBlob(shortName: string, buffer: Buffer): string {
+    const fullName = this._harRelativePath(shortName);
+    if (this._writtenContentEntries.has(shortName))
+      return fullName;
     if (!this._writtenContentEntries.size)
       this._fs.mkdir(this._resourcesDir);
-    this._writtenContentEntries.add(sha1);
-    this._fs.writeFile(path.join(this._resourcesDir, sha1), buffer, true /* skipIfExists */);
+    this._writtenContentEntries.add(shortName);
+    this._fs.writeFile(path.join(this._resourcesDir, shortName), buffer, true /* skipIfExists */);
+    return fullName;
   }
 
-  onContentBlobAppend(sha1: string, text: string) {
+  onContentBlobAppend(shortName: string, text: string) {
+    const fullName = this._harRelativePath(shortName);
     if (!this._writtenContentEntries.size)
       this._fs.mkdir(this._resourcesDir);
-    this._writtenContentEntries.add(sha1);
-    this._fs.appendFile(path.join(this._resourcesDir, sha1), text);
+    this._writtenContentEntries.add(shortName);
+    this._fs.appendFile(path.join(this._resourcesDir, shortName), text);
+    return fullName;
+  }
+
+  private _harRelativePath(shortName: string): string {
+    return this._relativeResourcesDir ? this._relativeResourcesDir + '/' + shortName : shortName;
   }
 
   private async _flush() {

packages/playwright-core/src/server/har/harTracer.ts

@@ -44,13 +44,12 @@ const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
 export interface HarTracerDelegate {
   onEntryStarted(entry: har.Entry): void;
   onEntryFinished(entry: har.Entry): void;
-  onContentBlob(sha1: string, buffer: Buffer): void;
-  onContentBlobAppend(sha1: string, text: string): void;
+  onContentBlob(shortName: string, buffer: Buffer): string;
+  onContentBlobAppend(shortName: string, text: string): string;
 }
 
 type HarTracerOptions = {
   content: 'omit' | 'attach' | 'embed';
-  includeTraceInfo: boolean;
   recordRequestOverrides: boolean;
   waitForContentOnStop: boolean;
   urlFilter?: string | RegExp;
@@ -446,7 +445,7 @@ export class HarTracer {
     const harEntry = createHarEntry(pageEntry?.id, method, url, page.mainFrame().guid, this._options, webSocket.wallTimeMs());
     harEntry._resourceType = 'websocket';
 
-    let sha1: string | undefined = undefined;
+    const shortName = createGuid() + '.jsonl';
     const recordMessage = (type: 'send' | 'receive', opcode: number, data: string, wallTimeMs: number) => {
       if (this._omitWebSocketFrames)
         return;
@@ -455,16 +454,9 @@ export class HarTracer {
         harEntry._webSocketMessages ??= [];
         harEntry._webSocketMessages.push(message);
       } else if (this._options.content === 'attach') {
-        if (!sha1) {
-          sha1 = createGuid() + '.jsonl';
-          if (this._options.includeTraceInfo)
-            harEntry.response.content._sha1 = sha1;
-          else
-            harEntry.response.content._file = sha1;
-        }
 
         if (this._started)
-          this._delegate.onContentBlobAppend(sha1, JSON.stringify(message) + '\n');
+          harEntry.response.content._file = this._delegate.onContentBlobAppend(shortName, JSON.stringify(message) + '\n');
       }
     };
 
@@ -561,13 +553,9 @@ export class HarTracer {
         content.encoding = 'base64';
       }
     } else if (this._options.content === 'attach') {
-      const sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
-      if (this._options.includeTraceInfo)
-        content._sha1 = sha1;
-      else
-        content._file = sha1;
+      const shortName = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
       if (this._started)
-        this._delegate.onContentBlob(sha1, buffer);
+        content._file = this._delegate.onContentBlob(shortName, buffer);
     }
   }
 
@@ -722,12 +710,8 @@ export class HarTracer {
       result.text = postData.toString();
 
     if (content === 'attach') {
-      const sha1 = calculateSha1(postData) + '.' + (mime.getExtension(contentType) || 'dat');
-      if (this._options.includeTraceInfo)
-        result._sha1 = sha1;
-      else
-        result._file = sha1;
-      this._delegate.onContentBlob(sha1, postData);
+      const shortName = calculateSha1(postData) + '.' + (mime.getExtension(contentType) || 'dat');
+      result._file = this._delegate.onContentBlob(shortName, postData);
     }
 
     if (contentType === 'application/x-www-form-urlencoded') {
@@ -778,8 +762,8 @@ function createHarEntry(pageRef: string | undefined, method: string, url: URL, f
       wait: -1,
       receive: -1
     },
-    _frameref: options.includeTraceInfo ? frameref : undefined,
-    _monotonicTime: options.includeTraceInfo ? monotonicTime() : undefined,
+    _frameref: frameref,
+    _monotonicTime: monotonicTime(),
   };
   return harEntry;
 }

packages/playwright-core/src/server/localUtils.ts

@@ -75,7 +75,7 @@ export async function zip(progress: Progress, stackSessions: Map<string, StackSe
         sourceFiles.add(file);
     }
     for (const sourceFile of sourceFiles)
-      addFile(sourceFile, 'resources/src@' + calculateSha1(sourceFile) + '.txt');
+      addFile(sourceFile, 'src/' + calculateSha1(sourceFile) + path.extname(sourceFile));
   }
 
   if (params.mode === 'write') {

packages/playwright-core/src/server/trace/recorder/snapshotter.ts

@@ -37,7 +37,7 @@ export type SnapshotterBlob = {
 };
 
 export interface SnapshotterDelegate {
-  onSnapshotterBlob(blob: SnapshotterBlob): void;
+  onSnapshotterBlob(blob: SnapshotterBlob): string;
   onFrameSnapshot(snapshot: FrameSnapshot): void;
 }
 
@@ -147,8 +147,8 @@ export class Snapshotter {
         if (typeof content === 'string') {
           const buffer = Buffer.from(content);
           const sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(contentType) || 'dat');
-          this._delegate.onSnapshotterBlob({ sha1, buffer });
-          snapshot.resourceOverrides.push({ url, sha1 });
+          const file = this._delegate.onSnapshotterBlob({ sha1, buffer });
+          snapshot.resourceOverrides.push({ url, file });
         } else {
           snapshot.resourceOverrides.push({ url, ref: content });
         }