From 0ff37d2321b15e9a74234c59aa4e37cf0f5a4d03 Mon Sep 17 00:00:00 2001 From: ik020 Date: Sat, 5 Sep 2026 15:05:12 +0500 Subject: [PATCH 1/2] Persist real face embeddings during actor indexing (#74 prep) Actor indexing computed a normalized face encoding per detection but never stored it: _actor_records() wrote a placeholder embedding=[0.0] into every StorageRecord, so the actor Chroma collection's vector index has never contained anything searchable. Matching only ever happened in-memory, within a single indexing run, via ActorIndexState.known_encodings, and was discarded once that run finished. This blocks issue #74 (find people using a reference image), which requires comparing a reference image's embedding against previously indexed faces across the whole repository. Fix: - Carry the computed encoding through into each detection dict. - _actor_records() now writes the real normalized encoding as the StorageRecord embedding instead of the placeholder. - Cluster-summary records (_actor_cluster_records) are left unchanged: a summary rolls up multiple detections and has no single face image to encode. Because this changes what's actually stored per detection, any existing generation with the actor modality enabled has placeholder vectors and is no longer valid. Bump INDEX_SCHEMA_VERSION 7 -> 8 so CompletedGenerationManifest's Literal[INDEX_SCHEMA_VERSION] check rejects old generations with a clear IndexSchemaError instead of silently treating placeholder-vector indexes as complete and searchable. Affected generations need to be re-indexed. Adds a new end-to-end test driving process_actor_samples through mocked detector/recognizer calls, asserting the stored embedding is the real normalized encoding rather than [0.0]. Updates the existing _actor_records test, which built detection dicts without the now-required encoding key. --- src/vidxp/capabilities/actor/indexing.py | 3 +- src/vidxp/core/contracts.py | 2 +- tests/test_indexing.py | 68 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/vidxp/capabilities/actor/indexing.py b/src/vidxp/capabilities/actor/indexing.py index 35da06c2..483694b9 100644 --- a/src/vidxp/capabilities/actor/indexing.py +++ b/src/vidxp/capabilities/actor/indexing.py @@ -74,7 +74,7 @@ def _actor_records( records.append( StorageRecord( source_id=source_id, - embedding=[0.0], + embedding=detection["encoding"], metadata={ **config.record_identity("actor", source_id), "detection_id": detection["detection_id"], @@ -209,6 +209,7 @@ def process_actor_samples( min(height, int(face[1] + face[3])), max(0, int(face[0])), ), + "encoding": encoding.tolist(), } ) state.processed_frames += 1 diff --git a/src/vidxp/core/contracts.py b/src/vidxp/core/contracts.py index c58d632f..34162bb8 100644 --- a/src/vidxp/core/contracts.py +++ b/src/vidxp/core/contracts.py @@ -11,7 +11,7 @@ from urllib.parse import quote -INDEX_SCHEMA_VERSION = 7 +INDEX_SCHEMA_VERSION = 8 MANIFEST_SCHEMA_VERSION = 2 diff --git a/tests/test_indexing.py b/tests/test_indexing.py index b60b2fb4..d72b53b8 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -444,6 +444,7 @@ def test_actor_records_preserve_stable_detection_metadata(self): "frame_index": 0, "timestamp": 0.0, "bbox": (1, 2, 3, 0), + "encoding": [0.6, 0.8], } ], config, @@ -455,6 +456,73 @@ def test_actor_records_preserve_stable_detection_metadata(self): "generation-1:actors:video-1:actor-cluster:1", ) self.assertEqual(records[0].metadata["bbox_top"], 1) + self.assertEqual(records[0].embedding, [0.6, 0.8]) + + def test_actor_indexing_persists_real_face_embeddings_not_placeholders(self): + import numpy as np + from unittest.mock import Mock + + from vidxp.capabilities.actor.indexing import ( + ActorIndexState, + process_actor_samples, + ) + from vidxp.core.video import FrameSample + + config = IndexConfig( + dataset="sample", + split="test", + run_id="actors", + video_id="video-1", + generation_id="generation-1", + enabled_modalities=("actor",), + ) + + raw_encoding = np.array([3.0, 4.0], dtype="float32") + expected_normalized = (raw_encoding / np.linalg.norm(raw_encoding)).tolist() + + detector = Mock() + detector.setInputSize = Mock() + detector.setScoreThreshold = Mock() + detector.detect.return_value = ( + True, + np.array([[10.0, 10.0, 20.0, 20.0]], dtype="float32"), + ) + + recognizer = Mock() + recognizer.alignCrop.return_value = np.zeros((112, 112, 3), dtype="uint8") + recognizer.feature.return_value = raw_encoding.reshape(1, -1) + + state = ActorIndexState( + models=Mock(detector=detector, recognizer=recognizer) + ) + storage = Mock() + + samples = [ + FrameSample( + frame_index=0, + timestamp=0.0, + frame=np.zeros((48, 48, 3), dtype="uint8"), + ), + ] + + process_actor_samples( + samples, + state=state, + config=config, + storage=storage, + cancellation=CancellationToken(), + ) + + self.assertEqual(storage.upsert.call_count, 1) + (_, records), _ = storage.upsert.call_args + self.assertEqual(len(records), 1) + record = records[0] + + self.assertIsNotNone(record.embedding) + self.assertNotEqual(list(record.embedding), [0.0]) + self.assertEqual(len(record.embedding), 2) + for actual, expected in zip(record.embedding, expected_normalized): + self.assertAlmostEqual(actual, expected, places=5) def test_actor_cluster_identity_is_unique_by_media_and_generation(self): def config(video_id, generation_id): From 7e44fd3c63332904af31d2b6a020c3a9ce5c1a4b Mon Sep 17 00:00:00 2001 From: ik020 Date: Tue, 15 Sep 2026 01:17:54 +0500 Subject: [PATCH 2/2] fix(actor): persist face embeddings during indexing --- src/vidxp/capabilities/actor/indexing.py | 8 ++- tests/test_indexing.py | 2 + tests/test_storage_integration.py | 80 ++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/vidxp/capabilities/actor/indexing.py b/src/vidxp/capabilities/actor/indexing.py index 483694b9..e53329ca 100644 --- a/src/vidxp/capabilities/actor/indexing.py +++ b/src/vidxp/capabilities/actor/indexing.py @@ -94,6 +94,7 @@ def _actor_records( def _actor_cluster_records( cluster_sizes: dict[str, int], cluster_ranges: dict[str, tuple[float, float]], + cluster_embeddings: dict[str, Any], config: IndexConfig, ) -> list[StorageRecord]: records = [] @@ -110,7 +111,7 @@ def _actor_cluster_records( records.append( StorageRecord( source_id=source_id, - embedding=[0.0], + embedding=[float(value) for value in cluster_embeddings[cluster_id]], metadata={ **config.record_identity("actor", source_id), "record_kind": "cluster_summary", @@ -252,6 +253,11 @@ def finalize_actor_index( cluster_id: state.cluster_ranges[cluster_id] for cluster_id in retained }, + { + cluster_id: state.known_encodings[index] + for index, cluster_id in enumerate(state.known_ids) + if cluster_id in retained + }, config, ), batch_size=config.storage_batch_size, diff --git a/tests/test_indexing.py b/tests/test_indexing.py index d72b53b8..0ae144e2 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -552,6 +552,7 @@ def test_actor_cluster_summaries_are_materialized_for_bounded_paging(self): records = _actor_cluster_records( {"cluster-1": 3}, {"cluster-1": (1.25, 4.5)}, + {"cluster-1": [0.6, 0.8]}, config, ) @@ -564,6 +565,7 @@ def test_actor_cluster_summaries_are_materialized_for_bounded_paging(self): self.assertEqual(records[0].metadata["detection_count"], 3) self.assertEqual(records[0].metadata["first_timestamp"], 1.25) self.assertEqual(records[0].metadata["last_timestamp"], 4.5) + self.assertEqual(records[0].embedding, [0.6, 0.8]) if __name__ == "__main__": diff --git a/tests/test_storage_integration.py b/tests/test_storage_integration.py index 6bdc7a3a..03c31e2a 100644 --- a/tests/test_storage_integration.py +++ b/tests/test_storage_integration.py @@ -8,6 +8,8 @@ StorageRecord, ) from vidxp.core.storage import IndexStorage, metadata_filter +from vidxp.capabilities.actor.config import actor_config +from vidxp.capabilities.actor.indexing import ActorIndexState, finalize_actor_index class ChromaStorageIntegrationTests(unittest.TestCase): @@ -130,6 +132,84 @@ def test_generation_scope_and_cleanup_use_chroma_in_filter(self): ["generation-2"], ) + + def test_actor_finalization_persists_cluster_summary_with_matching_embedding_dimension(self): + import numpy as np + from unittest.mock import Mock + + with TemporaryDirectory() as directory: + config = IndexConfig( + dataset="sample", + split="test", + run_id="actors", + video_id="video-1", + generation_id="generation-1", + enabled_modalities=("actor",), + storage_directory=directory, + ) + + centroid = np.zeros(128, dtype="float32") + centroid[0] = 1.0 + + cluster_id = "generation-1:actors:video-1:actor-cluster:1" + + state = ActorIndexState( + models=Mock(), + known_ids=[cluster_id], + known_encodings=[centroid], + cluster_sizes={cluster_id: 4}, + cluster_ranges={cluster_id: (1.0, 3.0)}, + ) + + with IndexStorage(config) as storage: + storage.upsert( + "actor", + [ + StorageRecord( + source_id="detection-1", + embedding=centroid.tolist(), + metadata={ + **config.record_identity( + "actor", "detection-1" + ), + "detection_id": "detection-1", + "cluster_id": cluster_id, + "frame_index": 0, + "timestamp": 1.0, + "bbox_top": 0, + "bbox_right": 10, + "bbox_bottom": 10, + "bbox_left": 0, + }, + ) + ], + batch_size=1, + cancellation=CancellationToken(), + ) + + finalize_actor_index( + state, + config=config, + storage=storage, + ) + + result = storage.collection("actor").get( + include=["embeddings", "metadatas"], + ) + + summary_embeddings = [ + embedding + for embedding, metadata in zip( + result["embeddings"], + result["metadatas"], + ) + if metadata.get("record_kind") == "cluster_summary" + ] + + self.assertEqual(len(summary_embeddings), 1) + self.assertEqual(len(summary_embeddings[0]), 128) + self.assertEqual(list(summary_embeddings[0]), centroid.tolist()) + def test_read_only_store_fails_closed_without_database_or_collection(self): with TemporaryDirectory() as directory: path = Path(directory)