diff --git a/packages/capture-protocol/package.json b/packages/capture-protocol/package.json
index 2dea2bf4d8..598b39e2ee 100644
--- a/packages/capture-protocol/package.json
+++ b/packages/capture-protocol/package.json
@@ -9,6 +9,7 @@
".": {
"types": "./src/index.ts",
"react-native": "./src/index.ts",
+ "browser": "./src/index.ts",
"import": "./dist/index.js",
"default": "./dist/index.js"
}
diff --git a/packages/capture-protocol/src/schema.test.ts b/packages/capture-protocol/src/schema.test.ts
index bd44354b74..3dbb6e8c7a 100644
--- a/packages/capture-protocol/src/schema.test.ts
+++ b/packages/capture-protocol/src/schema.test.ts
@@ -114,7 +114,7 @@ describe('capture manifests', () => {
).toThrow()
})
- test('rejects oversized or structurally inconsistent surface meshes', () => {
+ test('accepts the native 20,000-face preview budget and rejects malformed or oversized meshes', () => {
const surfaceMesh = {
version: 1,
coordinateSystem: 'arkit-world',
@@ -138,9 +138,19 @@ describe('capture manifests', () => {
streams: { surfaceMesh: { kind: 'surface-mesh', mesh } },
})
+ const atBudget = normalizeCaptureSessionManifest(
+ manifest({ ...surfaceMesh, faceCount: 20_000, indices: surfaceMesh.indices.repeat(20_000) }),
+ )
+ expect(atBudget.streams[0]?.inline).toMatchObject({ faceCount: 20_000 })
expect(() =>
- normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, faceCount: 6_001 })),
- ).toThrow()
+ normalizeCaptureSessionManifest(
+ manifest({
+ ...surfaceMesh,
+ faceCount: 20_001,
+ indices: surfaceMesh.indices.repeat(20_001),
+ }),
+ ),
+ ).toThrow('<=20000')
expect(() =>
normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, positions: 'AAAA' })),
).toThrow('decoded bytes')
diff --git a/packages/capture-protocol/src/schema.ts b/packages/capture-protocol/src/schema.ts
index 7ff16ea261..012e7b5fa2 100644
--- a/packages/capture-protocol/src/schema.ts
+++ b/packages/capture-protocol/src/schema.ts
@@ -52,7 +52,7 @@ export const ArkitPointCloudPayloadSchema = PointCloudPayloadSchema.safeExtend({
})
const MAX_SURFACE_MESH_VERTICES = 65_535
-const MAX_SURFACE_MESH_FACES = 6_000
+const MAX_SURFACE_MESH_FACES = 20_000
export const SurfaceMeshPayloadSchema = z
.object({
@@ -69,7 +69,10 @@ export const SurfaceMeshPayloadSchema = z
indexEncoding: z.literal('uint16x3-base64-little-endian'),
positions: z.string().min(1).max(524_280),
colors: z.string().min(1).max(262_140),
- indices: z.string().min(1).max(48_000),
+ indices: z
+ .string()
+ .min(1)
+ .max(MAX_SURFACE_MESH_FACES * 8),
})
.superRefine((payload, context) => {
if (payload.vertexCount > payload.faceCount * 3) {
diff --git a/packages/capture-viewer/README.md b/packages/capture-viewer/README.md
index 01c04f61d5..f6a63e3f01 100644
--- a/packages/capture-viewer/README.md
+++ b/packages/capture-viewer/README.md
@@ -31,3 +31,20 @@ enables them. Persisted values in the scan node's `layers` map always override t
without host defaults, every available layer remains visible for backwards compatibility.
Hidden sessions and layers are unmounted rather than only made visually transparent, so they stop
raycasting, artifact work, animation, and live packet subscriptions while disabled.
+
+## Local surface previews
+
+`@pascal-app/capture-viewer/preview` exports `createSurfaceMeshGeometry` and `createClayMatcap`
+without importing the React viewer runtime. A host can render a locally saved surface immediately,
+before its archive is uploaded. Browser and React Native exports resolve source, so an embedded
+DOM bundle does not depend on generated workspace `dist` files.
+
+The geometry decoder uses the shared capture-protocol validator, including the native
+20,000-face budget, byte lengths, and index bounds. It returns `null` for invalid input.
+The host owns the returned geometry and matcap texture and must dispose them on teardown.
+
+Direct `CaptureStreamLayer` consumers can pass
+`meshPresentation={{ previewMaterial: 'clay', dollhouse: true }}`. Clay replaces preliminary
+vertex colors; dollhouse enables front-face rendering for surface previews and room models,
+revealing inward-facing room surfaces from outside. It changes per-instance materials, not
+geometry or loader-cached materials. Omitting these options preserves the existing presentation.
diff --git a/packages/capture-viewer/package.json b/packages/capture-viewer/package.json
index 4829d0fa15..9b7db8a290 100644
--- a/packages/capture-viewer/package.json
+++ b/packages/capture-viewer/package.json
@@ -10,10 +10,18 @@
"types": "./dist/index.d.ts",
"import": "./dist/index.js",
"default": "./dist/index.js"
+ },
+ "./preview": {
+ "types": "./src/preview.ts",
+ "react-native": "./src/preview.ts",
+ "browser": "./src/preview.ts",
+ "import": "./dist/preview.js",
+ "default": "./dist/preview.js"
}
},
"files": [
"dist",
+ "src",
"README.md"
],
"scripts": {
diff --git a/packages/capture-viewer/src/capture-runtime.tsx b/packages/capture-viewer/src/capture-runtime.tsx
index e692334ba6..31cb22c909 100644
--- a/packages/capture-viewer/src/capture-runtime.tsx
+++ b/packages/capture-viewer/src/capture-runtime.tsx
@@ -44,9 +44,15 @@ import {
} from './stream-rendering'
import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory'
+export type CaptureMeshPresentation = {
+ dollhouse?: boolean
+ previewMaterial?: 'clay' | 'recorded'
+}
+
export type CaptureStreamRendererProps = {
artifactUrl: string | null
descriptor: CaptureSessionDescriptor
+ meshPresentation?: CaptureMeshPresentation
packets: readonly CaptureStreamPacket[]
scan: ScanNode
source: CaptureSource
@@ -238,6 +244,7 @@ function captureStreamRenderKey(stream: CaptureStreamDescriptor): string {
export function CaptureStreamLayer({
descriptor,
+ meshPresentation,
packets,
renderers,
scan,
@@ -295,6 +302,7 @@ export function CaptureStreamLayer({
)
} else if (layerKey === 'surfaceMesh') {
- content =
+ content = (
+
+ )
}
if (!(content && frameMatrix)) return content
return (
diff --git a/packages/capture-viewer/src/index.ts b/packages/capture-viewer/src/index.ts
index 93b16c1719..7cbf662e18 100644
--- a/packages/capture-viewer/src/index.ts
+++ b/packages/capture-viewer/src/index.ts
@@ -1,5 +1,6 @@
export { rewriteLoopbackAssetUrl } from './asset-url'
export {
+ type CaptureMeshPresentation,
CaptureRuntime,
type CaptureRuntimeErrorContext,
type CaptureRuntimeProps,
diff --git a/packages/capture-viewer/src/layers/clay-matcap.ts b/packages/capture-viewer/src/layers/clay-matcap.ts
new file mode 100644
index 0000000000..8b60baf8a3
--- /dev/null
+++ b/packages/capture-viewer/src/layers/clay-matcap.ts
@@ -0,0 +1,29 @@
+import { DataTexture, LinearFilter, RGBAFormat, SRGBColorSpace } from 'three'
+
+export function createClayMatcap(): DataTexture {
+ const size = 128
+ const pixels = new Uint8Array(size * size * 4)
+ for (let row = 0; row < size; row += 1) {
+ for (let column = 0; column < size; column += 1) {
+ const x = (column / (size - 1)) * 2 - 1
+ const y = (row / (size - 1)) * 2 - 1
+ const z = Math.sqrt(Math.max(0, 1 - x * x - y * y))
+ const gloss = Math.exp(-((x + 0.34) ** 2 / 0.045 + (y - 0.42) ** 2 / 0.075))
+ const rim = Math.exp(-((x - 0.65) ** 2 / 0.025 + (y + 0.15) ** 2 / 0.5)) * 0.3
+ const base = [0.3 + (1 - z) * 0.22, 0.35 + (x + 1) * 0.16 + z * 0.12, 0.64 + z * 0.2]
+ const offset = (row * size + column) * 4
+ for (let channel = 0; channel < 3; channel += 1) {
+ pixels[offset + channel] = Math.round(
+ Math.min(1, base[channel]! + gloss * 0.58 + rim) * 255,
+ )
+ }
+ pixels[offset + 3] = 255
+ }
+ }
+ const texture = new DataTexture(pixels, size, size, RGBAFormat)
+ texture.colorSpace = SRGBColorSpace
+ texture.minFilter = LinearFilter
+ texture.magFilter = LinearFilter
+ texture.needsUpdate = true
+ return texture
+}
diff --git a/packages/capture-viewer/src/layers/room-model-layer.tsx b/packages/capture-viewer/src/layers/room-model-layer.tsx
index a91dee9a87..9f4621b9bb 100644
--- a/packages/capture-viewer/src/layers/room-model-layer.tsx
+++ b/packages/capture-viewer/src/layers/room-model-layer.tsx
@@ -3,17 +3,19 @@
import { useGLTFKTX2 } from '@pascal-app/viewer'
import { useLoader } from '@react-three/fiber'
import { useEffect, useMemo } from 'react'
-import type { Material, Mesh, Object3D } from 'three'
+import { DoubleSide, FrontSide, type Material, type Mesh, type Object3D } from 'three'
import { USDLoader } from 'three/addons/loaders/USDLoader.js'
import { rewriteLoopbackAssetUrl } from '../asset-url'
import type { CaptureModelFormat } from '../stream-rendering'
export function CaptureRoomModel({
+ dollhouse,
format,
mediaType,
opacity = 100,
url,
}: {
+ dollhouse?: boolean
format?: CaptureModelFormat
mediaType: string
opacity?: number
@@ -24,24 +26,40 @@ export function CaptureRoomModel({
mediaType === 'model/vnd.usdz+zip' ||
url.toLowerCase().endsWith('.usdz')
) {
- return
+ return
}
- return
+ return
}
-function UsdzRoomModel({ opacity, url }: { opacity: number; url: string }) {
+function UsdzRoomModel({
+ dollhouse,
+ opacity,
+ url,
+}: {
+ dollhouse?: boolean
+ opacity: number
+ url: string
+}) {
const source = useLoader(USDLoader, rewriteLoopbackAssetUrl(url))
- const model = useClonedModel(source, opacity)
+ const model = useClonedModel(source, opacity, dollhouse)
return
}
-function GlbRoomModel({ opacity, url }: { opacity: number; url: string }) {
+function GlbRoomModel({
+ dollhouse,
+ opacity,
+ url,
+}: {
+ dollhouse?: boolean
+ opacity: number
+ url: string
+}) {
const gltf = useGLTFKTX2(rewriteLoopbackAssetUrl(url)) as { scene: Object3D }
- const model = useClonedModel(gltf.scene, opacity)
+ const model = useClonedModel(gltf.scene, opacity, dollhouse)
return
}
-function useClonedModel(source: Object3D, opacity: number): Object3D {
+function useClonedModel(source: Object3D, opacity: number, dollhouse?: boolean): Object3D {
const model = useMemo(() => {
const clone = source.clone(true)
clone.traverse((child) => {
@@ -50,9 +68,12 @@ function useClonedModel(source: Object3D, opacity: number): Object3D {
mesh.material = Array.isArray(mesh.material)
? mesh.material.map((material) => material.clone())
: mesh.material.clone()
+ for (const material of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) {
+ if (dollhouse !== undefined) material.side = dollhouse ? FrontSide : DoubleSide
+ }
})
return clone
- }, [source])
+ }, [source, dollhouse])
useEffect(() => {
const normalizedOpacity = opacity / 100
diff --git a/packages/capture-viewer/src/layers/surface-mesh-data.ts b/packages/capture-viewer/src/layers/surface-mesh-data.ts
new file mode 100644
index 0000000000..70141ed9ee
--- /dev/null
+++ b/packages/capture-viewer/src/layers/surface-mesh-data.ts
@@ -0,0 +1,83 @@
+import { SurfaceMeshPayloadSchema } from '@pascal-app/capture-protocol'
+import { BufferGeometry, Float32BufferAttribute, Uint16BufferAttribute } from 'three'
+
+export type SurfaceMeshData = {
+ colors: Float32Array
+ indices: Uint16Array
+ positions: Float32Array
+}
+
+export function createSurfaceMeshGeometry(value: unknown): BufferGeometry | null {
+ const data = buildSurfaceMeshData(value)
+ if (!data) return null
+ const geometry = new BufferGeometry()
+ geometry.setAttribute('position', new Float32BufferAttribute(data.positions, 3))
+ geometry.setAttribute('color', new Float32BufferAttribute(data.colors, 3))
+ geometry.setIndex(new Uint16BufferAttribute(data.indices, 1))
+ geometry.computeVertexNormals()
+ geometry.computeBoundingSphere()
+ return geometry
+}
+
+export function buildSurfaceMeshData(value: unknown): SurfaceMeshData | null {
+ const parsed = SurfaceMeshPayloadSchema.safeParse(value)
+ if (!parsed.success) return null
+ const payload = parsed.data
+ const positionBytes = decodeBase64(payload.positions)
+ const colorBytes = decodeBase64(payload.colors)
+ const indexBytes = decodeBase64(payload.indices)
+ if (
+ positionBytes.byteLength !== payload.vertexCount * 3 * 2 ||
+ colorBytes.byteLength !== payload.vertexCount * 3 ||
+ indexBytes.byteLength !== payload.faceCount * 3 * 2
+ ) {
+ return null
+ }
+
+ const positions = new Float32Array(payload.vertexCount * 3)
+ const colors = new Float32Array(payload.vertexCount * 3)
+ const indices = new Uint16Array(payload.faceCount * 3)
+ const positionView = new DataView(
+ positionBytes.buffer,
+ positionBytes.byteOffset,
+ positionBytes.byteLength,
+ )
+ const indexView = new DataView(indexBytes.buffer, indexBytes.byteOffset, indexBytes.byteLength)
+ for (let index = 0; index < payload.vertexCount; index += 1) {
+ for (let axis = 0; axis < 3; axis += 1) {
+ const offset = index * 3 + axis
+ const minimum = payload.boundsMin[axis] ?? 0
+ const maximum = payload.boundsMax[axis] ?? minimum
+ const quantized = positionView.getUint16(offset * 2, true)
+ positions[offset] = minimum + (quantized / 65_535) * (maximum - minimum)
+ colors[offset] = (colorBytes[offset] ?? 0) / 255
+ }
+ }
+ for (let index = 0; index < indices.length; index += 1) {
+ const vertexIndex = indexView.getUint16(index * 2, true)
+ if (vertexIndex >= payload.vertexCount) return null
+ indices[index] = vertexIndex
+ }
+ return { colors, indices, positions }
+}
+
+function decodeBase64(value: string): Uint8Array {
+ const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+ const clean = value.replace(/\s/g, '')
+ const padding = clean.endsWith('==') ? 2 : clean.endsWith('=') ? 1 : 0
+ const outputLength = Math.floor((clean.length * 3) / 4) - padding
+ const output = new Uint8Array(Math.max(0, outputLength))
+ let outputIndex = 0
+ for (let index = 0; index < clean.length; index += 4) {
+ const a = alphabet.indexOf(clean[index] ?? '')
+ const b = alphabet.indexOf(clean[index + 1] ?? '')
+ const c = clean[index + 2] === '=' ? 0 : alphabet.indexOf(clean[index + 2] ?? '')
+ const d = clean[index + 3] === '=' ? 0 : alphabet.indexOf(clean[index + 3] ?? '')
+ if (a < 0 || b < 0 || c < 0 || d < 0) return new Uint8Array()
+ const bits = (a << 18) | (b << 12) | (c << 6) | d
+ if (outputIndex < output.length) output[outputIndex++] = (bits >> 16) & 0xff
+ if (outputIndex < output.length) output[outputIndex++] = (bits >> 8) & 0xff
+ if (outputIndex < output.length) output[outputIndex++] = bits & 0xff
+ }
+ return output
+}
diff --git a/packages/capture-viewer/src/layers/surface-mesh-layer.tsx b/packages/capture-viewer/src/layers/surface-mesh-layer.tsx
index f8cf463608..d5b9ff0308 100644
--- a/packages/capture-viewer/src/layers/surface-mesh-layer.tsx
+++ b/packages/capture-viewer/src/layers/surface-mesh-layer.tsx
@@ -1,96 +1,39 @@
'use client'
-import { SurfaceMeshPayloadSchema } from '@pascal-app/capture-protocol'
import { useEffect, useMemo } from 'react'
-import { BufferGeometry, DoubleSide, Float32BufferAttribute, Uint16BufferAttribute } from 'three'
+import { DoubleSide, FrontSide } from 'three'
+import { createClayMatcap } from './clay-matcap'
+import { createSurfaceMeshGeometry } from './surface-mesh-data'
-export type SurfaceMeshData = {
- colors: Float32Array
- indices: Uint16Array
- positions: Float32Array
-}
+export { buildSurfaceMeshData, type SurfaceMeshData } from './surface-mesh-data'
-export function CaptureSurfaceMeshLayer({ inline }: { inline: unknown }) {
- const data = useMemo(() => buildSurfaceMeshData(inline), [inline])
- const geometry = useMemo(() => {
- if (!data) return null
- const next = new BufferGeometry()
- next.setAttribute('position', new Float32BufferAttribute(data.positions, 3))
- next.setAttribute('color', new Float32BufferAttribute(data.colors, 3))
- next.setIndex(new Uint16BufferAttribute(data.indices, 1))
- next.computeVertexNormals()
- next.computeBoundingSphere()
- return next
- }, [data])
+export function CaptureSurfaceMeshLayer({
+ inline,
+ dollhouse = false,
+ appearance = 'recorded',
+}: {
+ inline: unknown
+ dollhouse?: boolean
+ appearance?: 'clay' | 'recorded'
+}) {
+ const matcap = useMemo(() => (appearance === 'clay' ? createClayMatcap() : null), [appearance])
+ useEffect(() => () => matcap?.dispose(), [matcap])
+ const geometry = useMemo(() => createSurfaceMeshGeometry(inline), [inline])
useEffect(() => () => geometry?.dispose(), [geometry])
if (!geometry) return null
return (
-
+ {matcap ? (
+
+ ) : (
+
+ )}
)
}
-
-export function buildSurfaceMeshData(value: unknown): SurfaceMeshData | null {
- const parsed = SurfaceMeshPayloadSchema.safeParse(value)
- if (!parsed.success) return null
- const payload = parsed.data
- const positionBytes = decodeBase64(payload.positions)
- const colorBytes = decodeBase64(payload.colors)
- const indexBytes = decodeBase64(payload.indices)
- if (
- positionBytes.byteLength !== payload.vertexCount * 3 * 2 ||
- colorBytes.byteLength !== payload.vertexCount * 3 ||
- indexBytes.byteLength !== payload.faceCount * 3 * 2
- ) {
- return null
- }
-
- const positions = new Float32Array(payload.vertexCount * 3)
- const colors = new Float32Array(payload.vertexCount * 3)
- const indices = new Uint16Array(payload.faceCount * 3)
- const positionView = new DataView(
- positionBytes.buffer,
- positionBytes.byteOffset,
- positionBytes.byteLength,
- )
- const indexView = new DataView(indexBytes.buffer, indexBytes.byteOffset, indexBytes.byteLength)
- for (let index = 0; index < payload.vertexCount; index += 1) {
- for (let axis = 0; axis < 3; axis += 1) {
- const offset = index * 3 + axis
- const minimum = payload.boundsMin[axis] ?? 0
- const maximum = payload.boundsMax[axis] ?? minimum
- const quantized = positionView.getUint16(offset * 2, true)
- positions[offset] = minimum + (quantized / 65_535) * (maximum - minimum)
- colors[offset] = (colorBytes[offset] ?? 0) / 255
- }
- }
- for (let index = 0; index < indices.length; index += 1) {
- const vertexIndex = indexView.getUint16(index * 2, true)
- if (vertexIndex >= payload.vertexCount) return null
- indices[index] = vertexIndex
- }
- return { colors, indices, positions }
-}
-
-function decodeBase64(value: string): Uint8Array {
- const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
- const clean = value.replace(/\s/g, '')
- const padding = clean.endsWith('==') ? 2 : clean.endsWith('=') ? 1 : 0
- const outputLength = Math.floor((clean.length * 3) / 4) - padding
- const output = new Uint8Array(Math.max(0, outputLength))
- let outputIndex = 0
- for (let index = 0; index < clean.length; index += 4) {
- const a = alphabet.indexOf(clean[index] ?? '')
- const b = alphabet.indexOf(clean[index + 1] ?? '')
- const c = clean[index + 2] === '=' ? 0 : alphabet.indexOf(clean[index + 2] ?? '')
- const d = clean[index + 3] === '=' ? 0 : alphabet.indexOf(clean[index + 3] ?? '')
- if (a < 0 || b < 0 || c < 0 || d < 0) return new Uint8Array()
- const bits = (a << 18) | (b << 12) | (c << 6) | d
- if (outputIndex < output.length) output[outputIndex++] = (bits >> 16) & 0xff
- if (outputIndex < output.length) output[outputIndex++] = (bits >> 8) & 0xff
- if (outputIndex < output.length) output[outputIndex++] = bits & 0xff
- }
- return output
-}
diff --git a/packages/capture-viewer/src/preview.ts b/packages/capture-viewer/src/preview.ts
new file mode 100644
index 0000000000..b1a5449481
--- /dev/null
+++ b/packages/capture-viewer/src/preview.ts
@@ -0,0 +1,6 @@
+export { createClayMatcap } from './layers/clay-matcap'
+export {
+ buildSurfaceMeshData,
+ createSurfaceMeshGeometry,
+ type SurfaceMeshData,
+} from './layers/surface-mesh-data'