From 8b85e4163c3c5a6b1da53b63fad16b764cab2c4c Mon Sep 17 00:00:00 2001
From: Pavel Feldman <pavel.feldman@gmail.com>
Date: Wed, 19 Aug 2026 16:40:55 -0700
Subject: [PATCH] feat(trace-viewer): webm-based film strip
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A new trace event, {type: 'video', file, width, height,
timestampOrigin}, points at a video bundled in the trace zip. The
viewer decodes it in the browser at load time — an offscreen <video>
seek-stepped at 2 thumbnails per second, capped at 120 — and renders
the same film strip lanes and hover preview that screencast-frame
JPEGs drive, progressively as thumbnails land. timestampOrigin maps
the video's own clock onto the trace timeline, so a sparse recording
(frames only where the screen changed) still lines up with actions.

Recorders that capture the screen as compressed video can now bundle
one webm instead of hundreds of JPEG frames; the screenshot-based
strip is unchanged and the two can coexist in one trace.
---
 packages/isomorphic/trace/entries.ts          |   1 +
 packages/isomorphic/trace/traceLoader.ts      |   1 +
 packages/isomorphic/trace/traceModel.ts       |   5 +
 packages/isomorphic/trace/traceModernizer.ts  |   4 +
 packages/trace-viewer/src/ui/filmStrip.tsx    |  88 ++++++++++--
 .../trace-viewer/src/ui/videoThumbnails.ts    | 127 ++++++++++++++++++
 packages/trace/src/trace.ts                   |   9 ++
 7 files changed, 225 insertions(+), 10 deletions(-)
 create mode 100644 packages/trace-viewer/src/ui/videoThumbnails.ts

diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.ts
index 09c71959f30b1..a762d285bdd54 100644
--- a/packages/isomorphic/trace/entries.ts
+++ b/packages/isomorphic/trace/entries.ts
@@ -39,6 +39,7 @@ export type ContextEntry = {
   actions: ActionEntry[];
   screenshots: trace.ScreenshotTraceEvent[];
   ariaSnapshots: trace.AriaSnapshotTraceEvent[];
+  videos: trace.VideoTraceEvent[];
   events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[];
   stdio: trace.StdioTraceEvent[];
   errors: trace.ErrorTraceEvent[];
diff --git a/packages/isomorphic/trace/traceLoader.ts b/packages/isomorphic/trace/traceLoader.ts
index 40c0b0e4ba010..9daa159473037 100644
--- a/packages/isomorphic/trace/traceLoader.ts
+++ b/packages/isomorphic/trace/traceLoader.ts
@@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry {
     actions: [],
     screenshots: [],
     ariaSnapshots: [],
+    videos: [],
     events: [],
     errors: [],
     stdio: [],
diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.ts
index 27185f56e72a1..459088a7e8305 100644
--- a/packages/isomorphic/trace/traceModel.ts
+++ b/packages/isomorphic/trace/traceModel.ts
@@ -66,6 +66,7 @@ export class TraceModel {
   readonly title?: string;
   readonly options: trace.BrowserContextEventOptions;
   readonly pages: PageEntry[];
+  readonly videos: trace.VideoTraceEvent[];
   readonly actions: ActionEntry[];
   readonly attachments: Attachment[];
   readonly visibleAttachments: Attachment[];
@@ -105,6 +106,7 @@ export class TraceModel {
     // Next call updates all timestamps for all events in library contexts, so it must be done first.
     this.actions = mergeActionsAndUpdateTiming(contexts);
     this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages));
+    this.videos = [];
     this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE);
     this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE);
     this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE);
@@ -126,6 +128,7 @@ export class TraceModel {
         this._screenshots.set(`${event.callId}/${event.phase}`, event);
       for (const event of context.ariaSnapshots || [])
         this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event);
+      this.videos.push(...(context.videos || []));
     }
     this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []);
     this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_'));
@@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number)
     for (const frame of page.screencastFrames)
       frame.timestamp += monotonicTimeDelta;
   }
+  for (const video of context.videos || [])
+    video.timestampOrigin += monotonicTimeDelta;
   for (const resource of context.resources) {
     if (resource._monotonicTime)
       resource._monotonicTime += monotonicTimeDelta;
diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.ts
index 48012f9b0d57f..f883aaa37f635 100644
--- a/packages/isomorphic/trace/traceModernizer.ts
+++ b/packages/isomorphic/trace/traceModernizer.ts
@@ -110,6 +110,10 @@ export class TraceModernizer {
         contextEntry.screenshots.push(event);
         break;
       }
+      case 'video': {
+        contextEntry.videos.push(event);
+        break;
+      }
       case 'aria-snapshot': {
         contextEntry.ariaSnapshots.push(event);
         break;
diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx
index a264b28d4c64a..37a704ad61c34 100644
--- a/packages/trace-viewer/src/ui/filmStrip.tsx
+++ b/packages/trace-viewer/src/ui/filmStrip.tsx
@@ -20,6 +20,8 @@ import * as React from 'react';
 import { useMeasure, upperBound } from '@web/uiUtils';
 import type { PageEntry } from '@isomorphic/trace/entries';
 import { useTraceModel } from './traceModelContext';
+import { useVideoThumbnails } from './videoThumbnails';
+import type { VideoThumbnail } from './videoThumbnails';
 
 export type FilmStripPreviewPoint = {
   x: number;
@@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{
   const [measure, ref] = useMeasure<HTMLDivElement>();
   const lanesRef = React.useRef<HTMLDivElement>(null);
 
-  let pageIndex = 0;
+  const video = model?.videos?.[0];
+  const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined);
+
+  let laneIndex = 0;
   if (lanesRef.current && previewPoint) {
     const bounds = lanesRef.current.getBoundingClientRect();
-    pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
+    laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
   }
 
-  const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames;
+  const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);
+  const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : [];
+
+  let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;
+  if (laneIndex < pageLanes.length)
+    previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;
+  else
+    previewFrames = videoLanes[laneIndex - pageLanes.length];
+
   let previewImage = undefined;
   let previewSize = undefined;
-  if (previewPoint !== undefined && screencastFrames && screencastFrames.length) {
+  if (previewPoint !== undefined && previewFrames && previewFrames.length) {
     const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;
-    previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1];
+    previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];
     const fitInto = {
       width: Math.min(800, (window.innerWidth / 2) | 0),
       height: Math.min(800, (window.innerHeight / 2) | 0),
@@ -58,14 +71,20 @@ export const FilmStrip: React.FunctionComponent<{
   }
 
   return <div className='film-strip' ref={ref}>
-    <div className='film-strip-lanes' ref={lanesRef}>{
-      model?.pages.map((page, index) => page.screencastFrames.length ? <FilmStripLane
+    <div className='film-strip-lanes' ref={lanesRef}>
+      {pageLanes.map((page, index) => <FilmStripLane
         boundaries={boundaries}
         page={page}
         width={measure.width}
         key={index}
-      /> : null)
-    }</div>
+      />)}
+      {videoLanes.map((thumbnails, index) => <VideoFilmStripLane
+        boundaries={boundaries}
+        thumbnails={thumbnails}
+        width={measure.width}
+        key={'video-' + index}
+      />)}
+    </div>
     {model && previewPoint && previewImage && previewSize &&
       <div className='film-strip-hover' style={{
         top: measure.bottom + 5,
@@ -73,12 +92,61 @@ export const FilmStrip: React.FunctionComponent<{
         width: previewSize.width,
         height: previewSize.height,
       }}>
-        <img src={model.createRelativeUrl(`file/${previewImage.file}`)} width={previewSize.width} height={previewSize.height} />
+        <img src={previewImage.url} width={previewSize.width} height={previewSize.height} />
       </div>
     }
   </div>;
 };
 
+const VideoFilmStripLane: React.FunctionComponent<{
+  boundaries: Boundaries,
+  thumbnails: VideoThumbnail[],
+  width: number,
+}> = ({ boundaries, thumbnails, width }) => {
+  const viewportSize = { width: 0, height: 0 };
+  for (const thumbnail of thumbnails) {
+    viewportSize.width = Math.max(viewportSize.width, thumbnail.width);
+    viewportSize.height = Math.max(viewportSize.height, thumbnail.height);
+  }
+  const frameSize = inscribe(viewportSize, tileSize);
+  const startTime = thumbnails[0].timestamp;
+  const endTime = thumbnails[thumbnails.length - 1].timestamp;
+
+  const boundariesDuration = boundaries.maximum - boundaries.minimum;
+  const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width;
+  const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width;
+  const effectiveWidth = (endTime - startTime) / boundariesDuration * width;
+  const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0;
+  const frameDuration = (endTime - startTime) / frameCount;
+
+  const frames: React.JSX.Element[] = [];
+  for (let i = 0; startTime && frameDuration && i < frameCount; ++i) {
+    const time = startTime + frameDuration * i;
+    const index = upperBound(thumbnails, time, timeComparator) - 1;
+    frames.push(<div className='film-strip-frame' key={i} style={{
+      width: frameSize.width,
+      height: frameSize.height,
+      backgroundImage: `url(${thumbnails[index].url})`,
+      backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+      margin: frameMargin,
+      marginRight: frameMargin,
+    }} />);
+  }
+  frames.push(<div className='film-strip-frame' key={frames.length} style={{
+    width: frameSize.width,
+    height: frameSize.height,
+    backgroundImage: `url(${thumbnails[thumbnails.length - 1].url})`,
+    backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+    margin: frameMargin,
+    marginRight: frameMargin,
+  }} />);
+
+  return <div className='film-strip-lane' style={{
+    marginLeft: gapLeft + 'px',
+    marginRight: gapRight + 'px',
+  }}>{frames}</div>;
+};
+
 const FilmStripLane: React.FunctionComponent<{
   boundaries: Boundaries,
   page: PageEntry,
diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.ts
new file mode 100644
index 0000000000000..ebaac09bf6b20
--- /dev/null
+++ b/packages/trace-viewer/src/ui/videoThumbnails.ts
@@ -0,0 +1,127 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import * as React from 'react';
+import type * as trace from '@trace/trace';
+
+export type VideoThumbnail = {
+  timestamp: number;
+  url: string;
+  width: number;
+  height: number;
+};
+
+type CacheEntry = {
+  thumbnails: VideoThumbnail[];
+  listeners: Set<() => void>;
+  started: boolean;
+};
+
+const cache = new Map<string, CacheEntry>();
+
+const maxThumbnails = 120;
+const thumbnailsPerSecond = 2;
+
+export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] {
+  const [, setVersion] = React.useState(0);
+  const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined;
+  React.useEffect(() => {
+    if (!entry)
+      return;
+    const listener = () => setVersion(version => version + 1);
+    entry.listeners.add(listener);
+    return () => {
+      entry.listeners.delete(listener);
+    };
+  }, [entry]);
+  return entry?.thumbnails ?? [];
+}
+
+function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {
+  let entry = cache.get(videoUrl);
+  if (!entry) {
+    entry = { thumbnails: [], listeners: new Set(), started: false };
+    cache.set(videoUrl, entry);
+  }
+  if (!entry.started) {
+    entry.started = true;
+    void generateThumbnails(entry, video, videoUrl).catch(() => {});
+  }
+  return entry;
+}
+
+async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise<void> {
+  const response = await fetch(videoUrl);
+  const blob = await response.blob();
+  const objectUrl = URL.createObjectURL(blob);
+  const element = document.createElement('video');
+  element.muted = true;
+  element.preload = 'auto';
+  element.src = objectUrl;
+  try {
+    await new Promise<void>((resolve, reject) => {
+      element.addEventListener('loadedmetadata', () => resolve(), { once: true });
+      element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true });
+    });
+    let duration = element.duration;
+    if (!isFinite(duration) || duration <= 0) {
+      element.currentTime = Number.MAX_SAFE_INTEGER;
+      await new Promise<void>(resolve => element.addEventListener('seeked', () => resolve(), { once: true }));
+      duration = element.duration;
+      if (!isFinite(duration) || duration <= 0)
+        return;
+    }
+    const width = element.videoWidth || video.width;
+    const height = element.videoHeight || video.height;
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+    const context = canvas.getContext('2d');
+    if (!context)
+      return;
+    const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond)));
+    const step = duration / count;
+    for (let i = 0; i <= count; ++i) {
+      const time = Math.min(i * step, Math.max(0, duration - 0.001));
+      await seek(element, time);
+      context.drawImage(element, 0, 0, width, height);
+      const frame = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8));
+      if (!frame)
+        continue;
+      entry.thumbnails.push({
+        timestamp: video.timestampOrigin + time * 1000,
+        url: URL.createObjectURL(frame),
+        width,
+        height,
+      });
+      for (const listener of entry.listeners)
+        listener();
+    }
+  } finally {
+    element.removeAttribute('src');
+    element.load();
+    URL.revokeObjectURL(objectUrl);
+  }
+}
+
+async function seek(element: HTMLVideoElement, time: number): Promise<void> {
+  if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2)
+    return;
+  await new Promise<void>(resolve => {
+    element.addEventListener('seeked', () => resolve(), { once: true });
+    element.currentTime = time;
+  });
+}
diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts
index 46aa729f63b93..e5317b8b0825b 100644
--- a/packages/trace/src/trace.ts
+++ b/packages/trace/src/trace.ts
@@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = {
   frameSwapWallTime?: number,
 };
 
+export type VideoTraceEvent = {
+  type: 'video',
+  file: string,
+  width: number,
+  height: number,
+  timestampOrigin: number,
+};
+
 export type ActionPhase = 'before' | 'action' | 'after';
 
 export type ScreenshotTraceEvent = {
@@ -233,6 +241,7 @@ export type ErrorTraceEvent = {
 export type TraceEvent =
     ContextCreatedTraceEvent |
     ScreencastFrameTraceEvent |
+    VideoTraceEvent |
     ScreenshotTraceEvent |
     AriaSnapshotTraceEvent |
     ActionTraceEvent |