diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.ts index 09c71959f30b1..a762d285bdd54 100644 --- a/packages/isomorphic/trace/entries.ts +++ b/packages/isomorphic/trace/entries.ts @@ -39,6 +39,7 @@ export type ContextEntry = { actions: ActionEntry[]; screenshots: trace.ScreenshotTraceEvent[]; ariaSnapshots: trace.AriaSnapshotTraceEvent[]; + videos: trace.VideoTraceEvent[]; events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[]; stdio: trace.StdioTraceEvent[]; errors: trace.ErrorTraceEvent[]; diff --git a/packages/isomorphic/trace/traceLoader.ts b/packages/isomorphic/trace/traceLoader.ts index 40c0b0e4ba010..9daa159473037 100644 --- a/packages/isomorphic/trace/traceLoader.ts +++ b/packages/isomorphic/trace/traceLoader.ts @@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry { actions: [], screenshots: [], ariaSnapshots: [], + videos: [], events: [], errors: [], stdio: [], diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.ts index 27185f56e72a1..459088a7e8305 100644 --- a/packages/isomorphic/trace/traceModel.ts +++ b/packages/isomorphic/trace/traceModel.ts @@ -66,6 +66,7 @@ export class TraceModel { readonly title?: string; readonly options: trace.BrowserContextEventOptions; readonly pages: PageEntry[]; + readonly videos: trace.VideoTraceEvent[]; readonly actions: ActionEntry[]; readonly attachments: Attachment[]; readonly visibleAttachments: Attachment[]; @@ -105,6 +106,7 @@ export class TraceModel { // Next call updates all timestamps for all events in library contexts, so it must be done first. this.actions = mergeActionsAndUpdateTiming(contexts); this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages)); + this.videos = []; this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE); this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE); this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE); @@ -126,6 +128,7 @@ export class TraceModel { this._screenshots.set(`${event.callId}/${event.phase}`, event); for (const event of context.ariaSnapshots || []) this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event); + this.videos.push(...(context.videos || [])); } this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []); this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_')); @@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number) for (const frame of page.screencastFrames) frame.timestamp += monotonicTimeDelta; } + for (const video of context.videos || []) + video.timestampOrigin += monotonicTimeDelta; for (const resource of context.resources) { if (resource._monotonicTime) resource._monotonicTime += monotonicTimeDelta; diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.ts index 48012f9b0d57f..f883aaa37f635 100644 --- a/packages/isomorphic/trace/traceModernizer.ts +++ b/packages/isomorphic/trace/traceModernizer.ts @@ -110,6 +110,10 @@ export class TraceModernizer { contextEntry.screenshots.push(event); break; } + case 'video': { + contextEntry.videos.push(event); + break; + } case 'aria-snapshot': { contextEntry.ariaSnapshots.push(event); break; diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx index a264b28d4c64a..37a704ad61c34 100644 --- a/packages/trace-viewer/src/ui/filmStrip.tsx +++ b/packages/trace-viewer/src/ui/filmStrip.tsx @@ -20,6 +20,8 @@ import * as React from 'react'; import { useMeasure, upperBound } from '@web/uiUtils'; import type { PageEntry } from '@isomorphic/trace/entries'; import { useTraceModel } from './traceModelContext'; +import { useVideoThumbnails } from './videoThumbnails'; +import type { VideoThumbnail } from './videoThumbnails'; export type FilmStripPreviewPoint = { x: number; @@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{ const [measure, ref] = useMeasure(); const lanesRef = React.useRef(null); - let pageIndex = 0; + const video = model?.videos?.[0]; + const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined); + + let laneIndex = 0; if (lanesRef.current && previewPoint) { const bounds = lanesRef.current.getBoundingClientRect(); - pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0; + laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0; } - const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames; + const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length); + const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : []; + + let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined; + if (laneIndex < pageLanes.length) + previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined; + else + previewFrames = videoLanes[laneIndex - pageLanes.length]; + let previewImage = undefined; let previewSize = undefined; - if (previewPoint !== undefined && screencastFrames && screencastFrames.length) { + if (previewPoint !== undefined && previewFrames && previewFrames.length) { const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width; - previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1]; + previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1]; const fitInto = { width: Math.min(800, (window.innerWidth / 2) | 0), height: Math.min(800, (window.innerHeight / 2) | 0), @@ -58,14 +71,20 @@ export const FilmStrip: React.FunctionComponent<{ } return
-
{ - model?.pages.map((page, index) => page.screencastFrames.length ? + {pageLanes.map((page, index) => : null) - }
+ />)} + {videoLanes.map((thumbnails, index) => )} +
{model && previewPoint && previewImage && previewSize &&
- +
} ; }; +const VideoFilmStripLane: React.FunctionComponent<{ + boundaries: Boundaries, + thumbnails: VideoThumbnail[], + width: number, +}> = ({ boundaries, thumbnails, width }) => { + const viewportSize = { width: 0, height: 0 }; + for (const thumbnail of thumbnails) { + viewportSize.width = Math.max(viewportSize.width, thumbnail.width); + viewportSize.height = Math.max(viewportSize.height, thumbnail.height); + } + const frameSize = inscribe(viewportSize, tileSize); + const startTime = thumbnails[0].timestamp; + const endTime = thumbnails[thumbnails.length - 1].timestamp; + + const boundariesDuration = boundaries.maximum - boundaries.minimum; + const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width; + const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width; + const effectiveWidth = (endTime - startTime) / boundariesDuration * width; + const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0; + const frameDuration = (endTime - startTime) / frameCount; + + const frames: React.JSX.Element[] = []; + for (let i = 0; startTime && frameDuration && i < frameCount; ++i) { + const time = startTime + frameDuration * i; + const index = upperBound(thumbnails, time, timeComparator) - 1; + frames.push(
); + } + frames.push(
); + + return
{frames}
; +}; + const FilmStripLane: React.FunctionComponent<{ boundaries: Boundaries, page: PageEntry, diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.ts new file mode 100644 index 0000000000000..ebaac09bf6b20 --- /dev/null +++ b/packages/trace-viewer/src/ui/videoThumbnails.ts @@ -0,0 +1,127 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import * as React from 'react'; +import type * as trace from '@trace/trace'; + +export type VideoThumbnail = { + timestamp: number; + url: string; + width: number; + height: number; +}; + +type CacheEntry = { + thumbnails: VideoThumbnail[]; + listeners: Set<() => void>; + started: boolean; +}; + +const cache = new Map(); + +const maxThumbnails = 120; +const thumbnailsPerSecond = 2; + +export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] { + const [, setVersion] = React.useState(0); + const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined; + React.useEffect(() => { + if (!entry) + return; + const listener = () => setVersion(version => version + 1); + entry.listeners.add(listener); + return () => { + entry.listeners.delete(listener); + }; + }, [entry]); + return entry?.thumbnails ?? []; +} + +function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry { + let entry = cache.get(videoUrl); + if (!entry) { + entry = { thumbnails: [], listeners: new Set(), started: false }; + cache.set(videoUrl, entry); + } + if (!entry.started) { + entry.started = true; + void generateThumbnails(entry, video, videoUrl).catch(() => {}); + } + return entry; +} + +async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise { + const response = await fetch(videoUrl); + const blob = await response.blob(); + const objectUrl = URL.createObjectURL(blob); + const element = document.createElement('video'); + element.muted = true; + element.preload = 'auto'; + element.src = objectUrl; + try { + await new Promise((resolve, reject) => { + element.addEventListener('loadedmetadata', () => resolve(), { once: true }); + element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true }); + }); + let duration = element.duration; + if (!isFinite(duration) || duration <= 0) { + element.currentTime = Number.MAX_SAFE_INTEGER; + await new Promise(resolve => element.addEventListener('seeked', () => resolve(), { once: true })); + duration = element.duration; + if (!isFinite(duration) || duration <= 0) + return; + } + const width = element.videoWidth || video.width; + const height = element.videoHeight || video.height; + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + if (!context) + return; + const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond))); + const step = duration / count; + for (let i = 0; i <= count; ++i) { + const time = Math.min(i * step, Math.max(0, duration - 0.001)); + await seek(element, time); + context.drawImage(element, 0, 0, width, height); + const frame = await new Promise(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8)); + if (!frame) + continue; + entry.thumbnails.push({ + timestamp: video.timestampOrigin + time * 1000, + url: URL.createObjectURL(frame), + width, + height, + }); + for (const listener of entry.listeners) + listener(); + } + } finally { + element.removeAttribute('src'); + element.load(); + URL.revokeObjectURL(objectUrl); + } +} + +async function seek(element: HTMLVideoElement, time: number): Promise { + if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2) + return; + await new Promise(resolve => { + element.addEventListener('seeked', () => resolve(), { once: true }); + element.currentTime = time; + }); +} diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts index 46aa729f63b93..e5317b8b0825b 100644 --- a/packages/trace/src/trace.ts +++ b/packages/trace/src/trace.ts @@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = { frameSwapWallTime?: number, }; +export type VideoTraceEvent = { + type: 'video', + file: string, + width: number, + height: number, + timestampOrigin: number, +}; + export type ActionPhase = 'before' | 'action' | 'after'; export type ScreenshotTraceEvent = { @@ -233,6 +241,7 @@ export type ErrorTraceEvent = { export type TraceEvent = ContextCreatedTraceEvent | ScreencastFrameTraceEvent | + VideoTraceEvent | ScreenshotTraceEvent | AriaSnapshotTraceEvent | ActionTraceEvent |