feat(trace-viewer): webm-based film strip

packages/isomorphic/trace/entries.ts

@@ -39,6 +39,7 @@ export type ContextEntry = {
   actions: ActionEntry[];
   screenshots: trace.ScreenshotTraceEvent[];
   ariaSnapshots: trace.AriaSnapshotTraceEvent[];
+  videos: trace.VideoTraceEvent[];
   events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[];
   stdio: trace.StdioTraceEvent[];
   errors: trace.ErrorTraceEvent[];

packages/isomorphic/trace/traceLoader.ts

@@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry {
     actions: [],
     screenshots: [],
     ariaSnapshots: [],
+    videos: [],
     events: [],
     errors: [],
     stdio: [],

packages/isomorphic/trace/traceModel.ts

@@ -66,6 +66,7 @@ export class TraceModel {
   readonly title?: string;
   readonly options: trace.BrowserContextEventOptions;
   readonly pages: PageEntry[];
+  readonly videos: trace.VideoTraceEvent[];
   readonly actions: ActionEntry[];
   readonly attachments: Attachment[];
   readonly visibleAttachments: Attachment[];
@@ -105,6 +106,7 @@ export class TraceModel {
     // Next call updates all timestamps for all events in library contexts, so it must be done first.
     this.actions = mergeActionsAndUpdateTiming(contexts);
     this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages));
+    this.videos = [];
     this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE);
     this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE);
     this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE);
@@ -126,6 +128,7 @@ export class TraceModel {
         this._screenshots.set(`${event.callId}/${event.phase}`, event);
       for (const event of context.ariaSnapshots || [])
         this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event);
+      this.videos.push(...(context.videos || []));
     }
     this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []);
     this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_'));
@@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number)
     for (const frame of page.screencastFrames)
       frame.timestamp += monotonicTimeDelta;
   }
+  for (const video of context.videos || [])
+    video.timestampOrigin += monotonicTimeDelta;
   for (const resource of context.resources) {
     if (resource._monotonicTime)
       resource._monotonicTime += monotonicTimeDelta;

packages/isomorphic/trace/traceModernizer.ts

@@ -110,6 +110,10 @@ export class TraceModernizer {
         contextEntry.screenshots.push(event);
         break;
       }
+      case 'video': {
+        contextEntry.videos.push(event);
+        break;
+      }
       case 'aria-snapshot': {
         contextEntry.ariaSnapshots.push(event);
         break;

packages/trace-viewer/src/ui/filmStrip.tsx

@@ -20,6 +20,8 @@ import * as React from 'react';
 import { useMeasure, upperBound } from '@web/uiUtils';
 import type { PageEntry } from '@isomorphic/trace/entries';
 import { useTraceModel } from './traceModelContext';
+import { useVideoThumbnails } from './videoThumbnails';
+import type { VideoThumbnail } from './videoThumbnails';
 
 export type FilmStripPreviewPoint = {
   x: number;
@@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{
   const [measure, ref] = useMeasure<HTMLDivElement>();
   const lanesRef = React.useRef<HTMLDivElement>(null);
 
-  let pageIndex = 0;
+  const video = model?.videos?.[0];
+  const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined);
+
+  let laneIndex = 0;
   if (lanesRef.current && previewPoint) {
     const bounds = lanesRef.current.getBoundingClientRect();
-    pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
+    laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
   }
 
-  const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames;
+  const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);
+  const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : [];
+
+  let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;
+  if (laneIndex < pageLanes.length)
+    previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;
+  else
+    previewFrames = videoLanes[laneIndex - pageLanes.length];
+
   let previewImage = undefined;
   let previewSize = undefined;
-  if (previewPoint !== undefined && screencastFrames && screencastFrames.length) {
+  if (previewPoint !== undefined && previewFrames && previewFrames.length) {
     const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;
-    previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1];
+    previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];
     const fitInto = {
       width: Math.min(800, (window.innerWidth / 2) | 0),
       height: Math.min(800, (window.innerHeight / 2) | 0),
@@ -58,27 +71,82 @@ export const FilmStrip: React.FunctionComponent<{
   }
 
   return <div className='film-strip' ref={ref}>
-    <div className='film-strip-lanes' ref={lanesRef}>{
-      model?.pages.map((page, index) => page.screencastFrames.length ? <FilmStripLane
+    <div className='film-strip-lanes' ref={lanesRef}>
+      {pageLanes.map((page, index) => <FilmStripLane
         boundaries={boundaries}
         page={page}
         width={measure.width}
         key={index}
-      /> : null)
-    }</div>
+      />)}
+      {videoLanes.map((thumbnails, index) => <VideoFilmStripLane
+        boundaries={boundaries}
+        thumbnails={thumbnails}
+        width={measure.width}
+        key={'video-' + index}
+      />)}
+    </div>
     {model && previewPoint && previewImage && previewSize &&
       <div className='film-strip-hover' style={{
         top: measure.bottom + 5,
         left: Math.min(previewPoint.x, measure.width - previewSize.width - 10),
         width: previewSize.width,
         height: previewSize.height,
       }}>
-        <img src={model.createRelativeUrl(`file/${previewImage.file}`)} width={previewSize.width} height={previewSize.height} />
+        <img src={previewImage.url} width={previewSize.width} height={previewSize.height} />
       </div>
     }
   </div>;
 };
 
+const VideoFilmStripLane: React.FunctionComponent<{
+  boundaries: Boundaries,
+  thumbnails: VideoThumbnail[],
+  width: number,
+}> = ({ boundaries, thumbnails, width }) => {
+  const viewportSize = { width: 0, height: 0 };
+  for (const thumbnail of thumbnails) {
+    viewportSize.width = Math.max(viewportSize.width, thumbnail.width);
+    viewportSize.height = Math.max(viewportSize.height, thumbnail.height);
+  }
+  const frameSize = inscribe(viewportSize, tileSize);
+  const startTime = thumbnails[0].timestamp;
+  const endTime = thumbnails[thumbnails.length - 1].timestamp;
+
+  const boundariesDuration = boundaries.maximum - boundaries.minimum;
+  const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width;
+  const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width;
+  const effectiveWidth = (endTime - startTime) / boundariesDuration * width;
+  const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0;
+  const frameDuration = (endTime - startTime) / frameCount;
+
+  const frames: React.JSX.Element[] = [];
+  for (let i = 0; startTime && frameDuration && i < frameCount; ++i) {
+    const time = startTime + frameDuration * i;
+    const index = upperBound(thumbnails, time, timeComparator) - 1;
+    frames.push(<div className='film-strip-frame' key={i} style={{
+      width: frameSize.width,
+      height: frameSize.height,
+      backgroundImage: `url(${thumbnails[index].url})`,
+      backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+      margin: frameMargin,
+      marginRight: frameMargin,
+    }} />);
+  }
+  frames.push(<div className='film-strip-frame' key={frames.length} style={{
+    width: frameSize.width,
+    height: frameSize.height,
+    backgroundImage: `url(${thumbnails[thumbnails.length - 1].url})`,
+    backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+    margin: frameMargin,
+    marginRight: frameMargin,
+  }} />);
+
+  return <div className='film-strip-lane' style={{
+    marginLeft: gapLeft + 'px',
+    marginRight: gapRight + 'px',
+  }}>{frames}</div>;
+};
+
 const FilmStripLane: React.FunctionComponent<{
   boundaries: Boundaries,
   page: PageEntry,

packages/trace-viewer/src/ui/videoThumbnails.ts

@@ -0,0 +1,127 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import * as React from 'react';
+import type * as trace from '@trace/trace';
+
+export type VideoThumbnail = {
+  timestamp: number;
+  url: string;
+  width: number;
+  height: number;
+};
+
+type CacheEntry = {
+  thumbnails: VideoThumbnail[];
+  listeners: Set<() => void>;
+  started: boolean;
+};
+
+const cache = new Map<string, CacheEntry>();
+
+const maxThumbnails = 120;
+const thumbnailsPerSecond = 2;
+
+export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] {
+  const [, setVersion] = React.useState(0);
+  const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined;
+  React.useEffect(() => {
+    if (!entry)
+      return;
+    const listener = () => setVersion(version => version + 1);
+    entry.listeners.add(listener);
+    return () => {
+      entry.listeners.delete(listener);
+    };
+  }, [entry]);
+  return entry?.thumbnails ?? [];
+}
+
+function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {
+  let entry = cache.get(videoUrl);
+  if (!entry) {
+    entry = { thumbnails: [], listeners: new Set(), started: false };
+    cache.set(videoUrl, entry);
+  }
+  if (!entry.started) {
+    entry.started = true;
+    void generateThumbnails(entry, video, videoUrl).catch(() => {});
+  }
+  return entry;
+}
+
+async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise<void> {
+  const response = await fetch(videoUrl);
+  const blob = await response.blob();
+  const objectUrl = URL.createObjectURL(blob);
+  const element = document.createElement('video');
+  element.muted = true;
+  element.preload = 'auto';
+  element.src = objectUrl;
+  try {
+    await new Promise<void>((resolve, reject) => {
+      element.addEventListener('loadedmetadata', () => resolve(), { once: true });
+      element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true });
+    });
+    let duration = element.duration;
+    if (!isFinite(duration) || duration <= 0) {
+      element.currentTime = Number.MAX_SAFE_INTEGER;
+      await new Promise<void>(resolve => element.addEventListener('seeked', () => resolve(), { once: true }));
+      duration = element.duration;
+      if (!isFinite(duration) || duration <= 0)
+        return;
+    }
+    const width = element.videoWidth || video.width;
+    const height = element.videoHeight || video.height;
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+    const context = canvas.getContext('2d');
+    if (!context)
+      return;
+    const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond)));
+    const step = duration / count;
+    for (let i = 0; i <= count; ++i) {
+      const time = Math.min(i * step, Math.max(0, duration - 0.001));
+      await seek(element, time);
+      context.drawImage(element, 0, 0, width, height);
+      const frame = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8));
+      if (!frame)
+        continue;
+      entry.thumbnails.push({
+        timestamp: video.timestampOrigin + time * 1000,
+        url: URL.createObjectURL(frame),
+        width,
+        height,
+      });
+      for (const listener of entry.listeners)
+        listener();
+    }
+  } finally {
+    element.removeAttribute('src');
+    element.load();
+    URL.revokeObjectURL(objectUrl);
+  }
+}
+
+async function seek(element: HTMLVideoElement, time: number): Promise<void> {
+  if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2)
+    return;
+  await new Promise<void>(resolve => {
+    element.addEventListener('seeked', () => resolve(), { once: true });
+    element.currentTime = time;
+  });
+}

packages/trace/src/trace.ts

@@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = {
   frameSwapWallTime?: number,
 };
 
+export type VideoTraceEvent = {
+  type: 'video',
+  file: string,
+  width: number,
+  height: number,
+  timestampOrigin: number,
+};
+
 export type ActionPhase = 'before' | 'action' | 'after';
 
 export type ScreenshotTraceEvent = {
@@ -233,6 +241,7 @@ export type ErrorTraceEvent = {
 export type TraceEvent =
     ContextCreatedTraceEvent |
     ScreencastFrameTraceEvent |
+    VideoTraceEvent |
     ScreenshotTraceEvent |
     AriaSnapshotTraceEvent |
     ActionTraceEvent |