feat(capture): shared local Clay previews and dollhouse rendering

packages/capture-protocol/package.json

@@ -9,6 +9,7 @@
     ".": {
       "types": "./src/index.ts",
       "react-native": "./src/index.ts",
+      "browser": "./src/index.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     }

packages/capture-protocol/src/schema.test.ts

@@ -114,7 +114,7 @@ describe('capture manifests', () => {
     ).toThrow()
   })
 
-  test('rejects oversized or structurally inconsistent surface meshes', () => {
+  test('accepts the native 20,000-face preview budget and rejects malformed or oversized meshes', () => {
     const surfaceMesh = {
       version: 1,
       coordinateSystem: 'arkit-world',
@@ -138,9 +138,19 @@ describe('capture manifests', () => {
       streams: { surfaceMesh: { kind: 'surface-mesh', mesh } },
     })
 
+    const atBudget = normalizeCaptureSessionManifest(
+      manifest({ ...surfaceMesh, faceCount: 20_000, indices: surfaceMesh.indices.repeat(20_000) }),
+    )
+    expect(atBudget.streams[0]?.inline).toMatchObject({ faceCount: 20_000 })
     expect(() =>
-      normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, faceCount: 6_001 })),
-    ).toThrow()
+      normalizeCaptureSessionManifest(
+        manifest({
+          ...surfaceMesh,
+          faceCount: 20_001,
+          indices: surfaceMesh.indices.repeat(20_001),
+        }),
+      ),
+    ).toThrow('<=20000')
     expect(() =>
       normalizeCaptureSessionManifest(manifest({ ...surfaceMesh, positions: 'AAAA' })),
     ).toThrow('decoded bytes')

packages/capture-protocol/src/schema.ts

@@ -52,7 +52,7 @@ export const ArkitPointCloudPayloadSchema = PointCloudPayloadSchema.safeExtend({
 })
 
 const MAX_SURFACE_MESH_VERTICES = 65_535
-const MAX_SURFACE_MESH_FACES = 6_000
+const MAX_SURFACE_MESH_FACES = 20_000
 
 export const SurfaceMeshPayloadSchema = z
   .object({
@@ -69,7 +69,10 @@ export const SurfaceMeshPayloadSchema = z
     indexEncoding: z.literal('uint16x3-base64-little-endian'),
     positions: z.string().min(1).max(524_280),
     colors: z.string().min(1).max(262_140),
-    indices: z.string().min(1).max(48_000),
+    indices: z
+      .string()
+      .min(1)
+      .max(MAX_SURFACE_MESH_FACES * 8),
   })
   .superRefine((payload, context) => {
     if (payload.vertexCount > payload.faceCount * 3) {

packages/capture-viewer/README.md

@@ -31,3 +31,20 @@ enables them. Persisted values in the scan node's `layers` map always override t
 without host defaults, every available layer remains visible for backwards compatibility.
 Hidden sessions and layers are unmounted rather than only made visually transparent, so they stop
 raycasting, artifact work, animation, and live packet subscriptions while disabled.
+
+## Local surface previews
+
+`@pascal-app/capture-viewer/preview` exports `createSurfaceMeshGeometry` and `createClayMatcap`
+without importing the React viewer runtime. A host can render a locally saved surface immediately,
+before its archive is uploaded. Browser and React Native exports resolve source, so an embedded
+DOM bundle does not depend on generated workspace `dist` files.
+
+The geometry decoder uses the shared capture-protocol validator, including the native
+20,000-face budget, byte lengths, and index bounds. It returns `null` for invalid input.
+The host owns the returned geometry and matcap texture and must dispose them on teardown.
+
+Direct `CaptureStreamLayer` consumers can pass
+`meshPresentation={{ previewMaterial: 'clay', dollhouse: true }}`. Clay replaces preliminary
+vertex colors; dollhouse enables front-face rendering for surface previews and room models,
+revealing inward-facing room surfaces from outside. It changes per-instance materials, not
+geometry or loader-cached materials. Omitting these options preserves the existing presentation.

packages/capture-viewer/package.json

@@ -10,10 +10,18 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
+    },
+    "./preview": {
+      "types": "./src/preview.ts",
+      "react-native": "./src/preview.ts",
+      "browser": "./src/preview.ts",
+      "import": "./dist/preview.js",
+      "default": "./dist/preview.js"
     }
   },
   "files": [
     "dist",
+    "src",
     "README.md"
   ],
   "scripts": {

packages/capture-viewer/src/capture-runtime.tsx

@@ -44,9 +44,15 @@ import {
 } from './stream-rendering'
 import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory'
 
+export type CaptureMeshPresentation = {
+  dollhouse?: boolean
+  previewMaterial?: 'clay' | 'recorded'
+}
+
 export type CaptureStreamRendererProps = {
   artifactUrl: string | null
   descriptor: CaptureSessionDescriptor
+  meshPresentation?: CaptureMeshPresentation
   packets: readonly CaptureStreamPacket[]
   scan: ScanNode
   source: CaptureSource
@@ -238,6 +244,7 @@ function captureStreamRenderKey(stream: CaptureStreamDescriptor): string {
 
 export function CaptureStreamLayer({
   descriptor,
+  meshPresentation,
   packets,
   renderers,
   scan,
@@ -295,6 +302,7 @@ export function CaptureStreamLayer({
       <Renderer
         artifactUrl={artifactUrl}
         descriptor={descriptor}
+        meshPresentation={meshPresentation}
         packets={packets}
         scan={scan}
         source={source}
@@ -310,6 +318,7 @@ export function CaptureStreamLayer({
   ) {
     content = (
       <CaptureRoomModel
+        dollhouse={meshPresentation?.dollhouse}
         format={captureModelFormat(stream.artifact) ?? undefined}
         mediaType={stream.artifact.mediaType}
         opacity={scan.opacity}
@@ -331,7 +340,13 @@ export function CaptureStreamLayer({
       />
     )
   } else if (layerKey === 'surfaceMesh') {
-    content = <CaptureSurfaceMeshLayer inline={payload} />
+    content = (
+      <CaptureSurfaceMeshLayer
+        appearance={meshPresentation?.previewMaterial}
+        dollhouse={meshPresentation?.dollhouse}
+        inline={payload}
+      />
+    )
   }
   if (!(content && frameMatrix)) return content
   return (

packages/capture-viewer/src/index.ts

@@ -1,5 +1,6 @@
 export { rewriteLoopbackAssetUrl } from './asset-url'
 export {
+  type CaptureMeshPresentation,
   CaptureRuntime,
   type CaptureRuntimeErrorContext,
   type CaptureRuntimeProps,

packages/capture-viewer/src/layers/clay-matcap.ts

@@ -0,0 +1,29 @@
+import { DataTexture, LinearFilter, RGBAFormat, SRGBColorSpace } from 'three'
+
+export function createClayMatcap(): DataTexture {
+  const size = 128
+  const pixels = new Uint8Array(size * size * 4)
+  for (let row = 0; row < size; row += 1) {
+    for (let column = 0; column < size; column += 1) {
+      const x = (column / (size - 1)) * 2 - 1
+      const y = (row / (size - 1)) * 2 - 1
+      const z = Math.sqrt(Math.max(0, 1 - x * x - y * y))
+      const gloss = Math.exp(-((x + 0.34) ** 2 / 0.045 + (y - 0.42) ** 2 / 0.075))
+      const rim = Math.exp(-((x - 0.65) ** 2 / 0.025 + (y + 0.15) ** 2 / 0.5)) * 0.3
+      const base = [0.3 + (1 - z) * 0.22, 0.35 + (x + 1) * 0.16 + z * 0.12, 0.64 + z * 0.2]
+      const offset = (row * size + column) * 4
+      for (let channel = 0; channel < 3; channel += 1) {
+        pixels[offset + channel] = Math.round(
+          Math.min(1, base[channel]! + gloss * 0.58 + rim) * 255,
+        )
+      }
+      pixels[offset + 3] = 255
+    }
+  }
+  const texture = new DataTexture(pixels, size, size, RGBAFormat)
+  texture.colorSpace = SRGBColorSpace
+  texture.minFilter = LinearFilter
+  texture.magFilter = LinearFilter
+  texture.needsUpdate = true
+  return texture
+}

packages/capture-viewer/src/layers/room-model-layer.tsx

@@ -3,17 +3,19 @@
 import { useGLTFKTX2 } from '@pascal-app/viewer'
 import { useLoader } from '@react-three/fiber'
 import { useEffect, useMemo } from 'react'
-import type { Material, Mesh, Object3D } from 'three'
+import { DoubleSide, FrontSide, type Material, type Mesh, type Object3D } from 'three'
 import { USDLoader } from 'three/addons/loaders/USDLoader.js'
 import { rewriteLoopbackAssetUrl } from '../asset-url'
 import type { CaptureModelFormat } from '../stream-rendering'
 
 export function CaptureRoomModel({
+  dollhouse,
   format,
   mediaType,
   opacity = 100,
   url,
 }: {
+  dollhouse?: boolean
   format?: CaptureModelFormat
   mediaType: string
   opacity?: number
@@ -24,24 +26,40 @@ export function CaptureRoomModel({
     mediaType === 'model/vnd.usdz+zip' ||
     url.toLowerCase().endsWith('.usdz')
   ) {
-    return <UsdzRoomModel opacity={opacity} url={url} />
+    return <UsdzRoomModel dollhouse={dollhouse} opacity={opacity} url={url} />
   }
-  return <GlbRoomModel opacity={opacity} url={url} />
+  return <GlbRoomModel dollhouse={dollhouse} opacity={opacity} url={url} />
 }
 
-function UsdzRoomModel({ opacity, url }: { opacity: number; url: string }) {
+function UsdzRoomModel({
+  dollhouse,
+  opacity,
+  url,
+}: {
+  dollhouse?: boolean
+  opacity: number
+  url: string
+}) {
   const source = useLoader(USDLoader, rewriteLoopbackAssetUrl(url))
-  const model = useClonedModel(source, opacity)
+  const model = useClonedModel(source, opacity, dollhouse)
   return <primitive object={model} />
 }
 
-function GlbRoomModel({ opacity, url }: { opacity: number; url: string }) {
+function GlbRoomModel({
+  dollhouse,
+  opacity,
+  url,
+}: {
+  dollhouse?: boolean
+  opacity: number
+  url: string
+}) {
   const gltf = useGLTFKTX2(rewriteLoopbackAssetUrl(url)) as { scene: Object3D }
-  const model = useClonedModel(gltf.scene, opacity)
+  const model = useClonedModel(gltf.scene, opacity, dollhouse)
   return <primitive object={model} />
 }
 
-function useClonedModel(source: Object3D, opacity: number): Object3D {
+function useClonedModel(source: Object3D, opacity: number, dollhouse?: boolean): Object3D {
   const model = useMemo(() => {
     const clone = source.clone(true)
     clone.traverse((child) => {
@@ -50,9 +68,12 @@ function useClonedModel(source: Object3D, opacity: number): Object3D {
       mesh.material = Array.isArray(mesh.material)
         ? mesh.material.map((material) => material.clone())
         : mesh.material.clone()
+      for (const material of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) {
+        if (dollhouse !== undefined) material.side = dollhouse ? FrontSide : DoubleSide
+      }
     })
     return clone
-  }, [source])
+  }, [source, dollhouse])
 
   useEffect(() => {
     const normalizedOpacity = opacity / 100

packages/capture-viewer/src/layers/surface-mesh-data.ts

@@ -0,0 +1,83 @@
+import { SurfaceMeshPayloadSchema } from '@pascal-app/capture-protocol'
+import { BufferGeometry, Float32BufferAttribute, Uint16BufferAttribute } from 'three'
+
+export type SurfaceMeshData = {
+  colors: Float32Array
+  indices: Uint16Array
+  positions: Float32Array
+}
+
+export function createSurfaceMeshGeometry(value: unknown): BufferGeometry | null {
+  const data = buildSurfaceMeshData(value)
+  if (!data) return null
+  const geometry = new BufferGeometry()
+  geometry.setAttribute('position', new Float32BufferAttribute(data.positions, 3))
+  geometry.setAttribute('color', new Float32BufferAttribute(data.colors, 3))
+  geometry.setIndex(new Uint16BufferAttribute(data.indices, 1))
+  geometry.computeVertexNormals()
+  geometry.computeBoundingSphere()
+  return geometry
+}
+
+export function buildSurfaceMeshData(value: unknown): SurfaceMeshData | null {
+  const parsed = SurfaceMeshPayloadSchema.safeParse(value)
+  if (!parsed.success) return null
+  const payload = parsed.data
+  const positionBytes = decodeBase64(payload.positions)
+  const colorBytes = decodeBase64(payload.colors)
+  const indexBytes = decodeBase64(payload.indices)
+  if (
+    positionBytes.byteLength !== payload.vertexCount * 3 * 2 ||
+    colorBytes.byteLength !== payload.vertexCount * 3 ||
+    indexBytes.byteLength !== payload.faceCount * 3 * 2
+  ) {
+    return null
+  }
+
+  const positions = new Float32Array(payload.vertexCount * 3)
+  const colors = new Float32Array(payload.vertexCount * 3)
+  const indices = new Uint16Array(payload.faceCount * 3)
+  const positionView = new DataView(
+    positionBytes.buffer,
+    positionBytes.byteOffset,
+    positionBytes.byteLength,
+  )
+  const indexView = new DataView(indexBytes.buffer, indexBytes.byteOffset, indexBytes.byteLength)
+  for (let index = 0; index < payload.vertexCount; index += 1) {
+    for (let axis = 0; axis < 3; axis += 1) {
+      const offset = index * 3 + axis
+      const minimum = payload.boundsMin[axis] ?? 0
+      const maximum = payload.boundsMax[axis] ?? minimum
+      const quantized = positionView.getUint16(offset * 2, true)
+      positions[offset] = minimum + (quantized / 65_535) * (maximum - minimum)
+      colors[offset] = (colorBytes[offset] ?? 0) / 255
+    }
+  }
+  for (let index = 0; index < indices.length; index += 1) {
+    const vertexIndex = indexView.getUint16(index * 2, true)
+    if (vertexIndex >= payload.vertexCount) return null
+    indices[index] = vertexIndex
+  }
+  return { colors, indices, positions }
+}
+
+function decodeBase64(value: string): Uint8Array {
+  const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+  const clean = value.replace(/\s/g, '')
+  const padding = clean.endsWith('==') ? 2 : clean.endsWith('=') ? 1 : 0
+  const outputLength = Math.floor((clean.length * 3) / 4) - padding
+  const output = new Uint8Array(Math.max(0, outputLength))
+  let outputIndex = 0
+  for (let index = 0; index < clean.length; index += 4) {
+    const a = alphabet.indexOf(clean[index] ?? '')
+    const b = alphabet.indexOf(clean[index + 1] ?? '')
+    const c = clean[index + 2] === '=' ? 0 : alphabet.indexOf(clean[index + 2] ?? '')
+    const d = clean[index + 3] === '=' ? 0 : alphabet.indexOf(clean[index + 3] ?? '')
+    if (a < 0 || b < 0 || c < 0 || d < 0) return new Uint8Array()
+    const bits = (a << 18) | (b << 12) | (c << 6) | d
+    if (outputIndex < output.length) output[outputIndex++] = (bits >> 16) & 0xff
+    if (outputIndex < output.length) output[outputIndex++] = (bits >> 8) & 0xff
+    if (outputIndex < output.length) output[outputIndex++] = bits & 0xff
+  }
+  return output
+}