diff --git a/src/commands/watch.rs b/src/commands/watch.rs index efb9273..762b9fb 100644 --- a/src/commands/watch.rs +++ b/src/commands/watch.rs @@ -34,11 +34,19 @@ pub fn run_watch( verbose: bool, ) -> Result<()> { let conn = crate::index::open_global_index()?; - // Load skip patterns from config - let skip_patterns: Option> = - crate::config::loader::load_config(config_path, None, None, None, None, false) - .ok() - .map(|c| c.rules_config.index_skip_files); + // Load skip patterns + brain embedding backend from config + let config = + crate::config::loader::load_config(config_path, None, None, None, None, false).ok(); + let skip_patterns: Option> = config + .as_ref() + .map(|c| c.rules_config.index_skip_files.clone()); + + // Resolve embedding backend + let brain_mode = config + .as_ref() + .map(|c| c.brain.embedding.to_string()) + .unwrap_or_else(|| "auto".to_string()); + crate::embed::resolve_backend(&brain_mode); let skip_ref: Option<&[String]> = skip_patterns.as_deref(); diff --git a/src/config/schema.rs b/src/config/schema.rs index d81d3c5..624f2b4 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -56,6 +56,9 @@ pub struct Config { /// Analysis configuration — dead-code detection, entry-point patterns. #[serde(default, skip_serializing_if = "is_default")] pub analysis: AnalysisConfig, + /// Brain Mode configuration — embedding backend selection. + #[serde(default, skip_serializing_if = "is_default")] + pub brain: BrainConfig, } /// Provider configuration. @@ -141,6 +144,7 @@ impl Default for Config { debt: crate::engine::debt_tracker::DebtConfig::default(), profile: None, analysis: AnalysisConfig::default(), + brain: BrainConfig::default(), } } } @@ -414,6 +418,62 @@ pub struct AnalysisConfig { pub entry_point_patterns: Vec, } +/// Brain Mode configuration — controls embedding backend for vector search. +/// +/// By default (`auto`), cora selects the best available backend at runtime: +/// pretrained 768d (if compiled with `pretrained-embed` feature) → hashing 256d fallback. +/// Users can force a specific backend via `.cora.yaml`. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct BrainConfig { + /// Embedding backend selection. + /// + /// - `"auto"` (default) — best available: pretrained → hashing + /// - `"hashing"` — force 256d hashing trick (zero dependency) + /// - `"pretrained"` — force nomic 768d (requires `--features pretrained-embed`) + /// + /// Invalid values fall back to `"auto"` with a warning. + #[serde(default, skip_serializing_if = "is_default")] + pub embedding: BrainEmbeddingMode, +} + +/// Embedding backend mode for Brain Mode. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum BrainEmbeddingMode { + /// Best available backend (pretrained if compiled, else hashing). + #[default] + Auto, + /// Force 256d hashing trick (zero dependency). + Hashing, + /// Force nomic 768d pretrained (requires feature flag). + Pretrained, +} + +impl std::fmt::Display for BrainEmbeddingMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Auto => write!(f, "auto"), + Self::Hashing => write!(f, "hashing"), + Self::Pretrained => write!(f, "pretrained"), + } + } +} + +impl std::str::FromStr for BrainEmbeddingMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "auto" => Ok(Self::Auto), + "hashing" => Ok(Self::Hashing), + "pretrained" => Ok(Self::Pretrained), + other => Err(format!( + "unknown brain.embedding value '{other}' — expected auto, hashing, or pretrained" + )), + } + } +} + fn is_default(val: &T) -> bool { *val == T::default() } diff --git a/src/embed/mod.rs b/src/embed/mod.rs index e1b35d3..f1a8c53 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -10,8 +10,10 @@ //! compiled into the binary via `include_bytes!` / `include_str!`. //! Higher quality at the cost of ~30 MB binary size. //! -//! The [`embed_code_dispatch`] function selects the best available backend at -//! compile time: pretrained-embed (768d) → hashing trick (256d) → FTS5-only. +//! The active backend is selected at **runtime** via [`resolve_backend`], +//! which reads the `brain.embedding` config value. At compile time, only +//! the availability of the pretrained path is gated by the `pretrained-embed` +//! feature flag. pub mod tokens; @@ -27,63 +29,162 @@ pub use tokens::EMBEDDING_DIM; #[cfg(feature = "pretrained-embed")] pub use token_vocab::{PRETRAINED_DIM, embed_code_pretrained}; -/// Returns the embedding dimensionality used by the active backend. +/// Runtime embedding backend selector. /// -/// - `pretrained-embed` feature → 768 -/// - default (hashing trick) → 256 -pub const fn active_dims() -> usize { - #[cfg(feature = "pretrained-embed")] - { - PRETRAINED_DIM +/// Resolved from `brain.embedding` config value at call time. +/// Falls back gracefully when a requested backend is not compiled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Backend { + /// 256-dim hashing trick (always available, zero dependency). + Hashing, + /// 768-dim nomic distilled pretrained (requires `pretrained-embed` feature). + #[allow(dead_code)] + Pretrained, +} + +impl Backend { + /// Returns the embedding dimensionality for this backend. + pub const fn dims(self) -> usize { + match self { + Self::Hashing => EMBEDDING_DIM, + #[cfg(feature = "pretrained-embed")] + Self::Pretrained => PRETRAINED_DIM, + #[cfg(not(feature = "pretrained-embed"))] + Self::Pretrained => EMBEDDING_DIM, // unreachable — resolve never returns Pretrained without feature + } } - #[cfg(not(feature = "pretrained-embed"))] - { - EMBEDDING_DIM + + /// Returns a human-readable label for this backend. + pub const fn provider_name(self) -> &'static str { + match self { + Self::Hashing => "hashing-trick (256d, static)", + #[cfg(feature = "pretrained-embed")] + Self::Pretrained => "nomic-embed-code (768d, pretrained)", + #[cfg(not(feature = "pretrained-embed"))] + Self::Pretrained => "hashing-trick (256d, pretrained not compiled)", + } } } -/// Returns a human-readable label for the active embedding provider. -pub const fn active_provider_name() -> &'static str { - #[cfg(feature = "pretrained-embed")] - { - "nomic-embed-code (768d, pretrained)" - } - #[cfg(not(feature = "pretrained-embed"))] - { - "hashing-trick (256d, static)" +/// Thread-local active backend — set once at index/brain-search time. +static ACTIVE_BACKEND: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Resolve the runtime backend from config string. +/// +/// Logic: +/// - `"auto"` → pretrained if compiled, else hashing +/// - `"hashing"` → always hashing +/// - `"pretrained"` → pretrained if compiled, else hashing + warning +/// +/// The result is cached process-wide via `OnceLock`. +pub fn resolve_backend(config_embedding: &str) -> Backend { + // Already resolved? Return cached value. + if let Some(&b) = ACTIVE_BACKEND.get() { + return b; } + + let backend = match config_embedding { + "hashing" => Backend::Hashing, + "pretrained" => { + #[cfg(feature = "pretrained-embed")] + { + Backend::Pretrained + } + #[cfg(not(feature = "pretrained-embed"))] + { + tracing::warn!( + "brain.embedding=pretrained but cora was not compiled with --features pretrained-embed; \ + falling back to hashing-trick 256d" + ); + Backend::Hashing + } + } + // "auto" or any unknown value + _ => { + #[cfg(feature = "pretrained-embed")] + { + Backend::Pretrained + } + #[cfg(not(feature = "pretrained-embed"))] + { + Backend::Hashing + } + } + }; + + let _ = ACTIVE_BACKEND.set(backend); + tracing::debug!( + config = config_embedding, + backend = ?backend, + "resolved embedding backend" + ); + backend +} + +/// Returns the embedding dimensionality used by the active backend. +/// +/// Convenience wrapper around `resolve_backend().dims()`. +pub fn active_dims() -> usize { + // Use a sensible default if resolve_backend hasn't been called yet. + ACTIVE_BACKEND.get().map(|b| b.dims()).unwrap_or_else(|| { + #[cfg(feature = "pretrained-embed")] + { + PRETRAINED_DIM + } + #[cfg(not(feature = "pretrained-embed"))] + { + EMBEDDING_DIM + } + }) +} + +/// Returns a human-readable label for the active embedding provider. +pub fn active_provider_name() -> &'static str { + ACTIVE_BACKEND + .get() + .map(|b| b.provider_name()) + .unwrap_or_else(|| { + #[cfg(feature = "pretrained-embed")] + { + "nomic-embed-code (768d, pretrained)" + } + #[cfg(not(feature = "pretrained-embed"))] + { + "hashing-trick (256d, static)" + } + }) } /// Embed a code snippet using the best available backend. /// /// Returns an f32 vector that can be passed directly to usearch. /// -/// - **Pretrained path** (`pretrained-embed` feature): tokenises → looks up -/// each token in the nomic vocabulary → accumulates int8 vectors → L2-normalises. -/// Returns 768-dim vector. -/// -/// - **Hashing-trick fallback**: tokenises → hashes each token into a -/// pseudo-random 256-dim vector → accumulates → L2-normalises. -/// Returns 256-dim vector. -/// -/// Both paths share the same [`tokenize_code`] tokenizer. +/// Dispatches to the backend set by [`resolve_backend`]. If no backend has +/// been explicitly resolved, falls back to compile-time default. pub fn embed_code_dispatch(code: &str) -> Vec { - #[cfg(feature = "pretrained-embed")] - { - embed_code_pretrained(code) - } - #[cfg(not(feature = "pretrained-embed"))] - { - let embedding = tokens::embed_code(code); - embedding.as_slice().iter().map(|&v| v as f32).collect() + let backend = ACTIVE_BACKEND.get().copied().unwrap_or_else(|| { + // Lazy resolve with "auto" if not yet set + resolve_backend("auto") + }); + + match backend { + Backend::Hashing => { + let embedding = tokens::embed_code(code); + embedding.as_slice().iter().map(|&v| v as f32).collect() + } + #[cfg(feature = "pretrained-embed")] + Backend::Pretrained => embed_code_pretrained(code), + #[cfg(not(feature = "pretrained-embed"))] + Backend::Pretrained => { + // Should never happen — resolve_backend never returns Pretrained without feature + let embedding = tokens::embed_code(code); + embedding.as_slice().iter().map(|&v| v as f32).collect() + } } } /// Whether the pretrained embedding backend is available (compile-time). -#[expect( - dead_code, - reason = "used by Phase 3+ features; embed module not yet wired at call sites" -)] +#[allow(dead_code)] pub const fn has_pretrained() -> bool { cfg!(feature = "pretrained-embed") } diff --git a/src/index/brain.rs b/src/index/brain.rs index d917235..49b892e 100644 --- a/src/index/brain.rs +++ b/src/index/brain.rs @@ -120,9 +120,15 @@ fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) { /// Embed all symbols for a project into the vector index. /// -/// Uses the best available embedding backend (selected at compile time): -/// - `pretrained-embed` → nomic-embed-code 768-dim vectors -/// - default → hashing-trick 256-dim vectors +/// Uses the embedding backend selected at runtime via [`resolve_backend`]: +/// - `"pretrained"` → nomic-embed-code 768-dim vectors +/// - `"hashing"` → hashing-trick 256-dim vectors +/// - `"auto"` → best available +/// +/// **Incremental**: Only symbols whose `name + signature` fingerprint has +/// changed since the last embed are re-embedded. This dramatically reduces +/// embedding time when a single file is modified (e.g. 10 changed symbols +/// out of 1100 total). /// /// Detects dimension mismatch between existing on-disk index and current /// backend, warning the user to re-index if dimensions changed. @@ -152,27 +158,65 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { cache.as_mut().unwrap() }; - let mut stmt = - conn.prepare("SELECT id, name, kind, signature FROM symbols WHERE project_id = ?1")?; - let rows: Vec<(i64, String, String, String)> = stmt + // ── Incremental: fetch stored fingerprints ────────────────────── + // Only re-embed symbols whose name+signature has changed. + let mut stmt = conn.prepare( + "SELECT id, name, kind, signature, embed_fingerprint \ + FROM symbols WHERE project_id = ?1", + )?; + let rows: Vec<(i64, String, String, String, Option)> = stmt .query_map(rusqlite::params![project_id], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) })? .filter_map(|r| r.ok()) .collect(); - // ── Parallel embedding computation (Rayon) ───────────────────────── - // embed_code_dispatch is pure + CPU-bound. usearch insert is serial. - let t_compute = std::time::Instant::now(); - let embedded: Vec<(i64, Vec)> = rows - .par_iter() - .map(|(sym_id, name, _kind, signature)| { + // Compute current fingerprints and filter to only changed symbols + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let changed: Vec<(i64, String)> = rows + .iter() + .filter_map(|(sym_id, name, _kind, signature, stored_fp)| { let text = if signature.is_empty() || signature == name { name.clone() } else { format!("{name} {signature}") }; - let vec = embed_code_dispatch(&text); + let mut hasher = DefaultHasher::new(); + text.hash(&mut hasher); + let current_fp = format!("{:016x}", hasher.finish()); + + if stored_fp.as_deref() == Some(¤t_fp) { + None // unchanged — skip + } else { + Some((*sym_id, text)) + } + }) + .collect(); + + let total_symbols = rows.len(); + let skipped = total_symbols - changed.len(); + if skipped > 0 { + tracing::info!( + "Incremental embed: {total_symbols} total, {skipped} unchanged (skipped), {} changed (re-embedding)", + changed.len() + ); + } + + // ── Parallel embedding computation (Rayon) ───────────────────────── + // embed_code_dispatch is pure + CPU-bound. usearch insert is serial. + let t_compute = std::time::Instant::now(); + let embedded: Vec<(i64, Vec)> = changed + .par_iter() + .map(|(sym_id, text)| { + let vec = embed_code_dispatch(text); (*sym_id, vec) }) .collect(); @@ -181,19 +225,24 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { // ── Serial usearch insert ──────────────────────────────────────── let t_insert = std::time::Instant::now(); let mut count = 0; - let mut new_ids: HashSet = HashSet::with_capacity(embedded.len()); + let mut new_ids: HashSet = HashSet::with_capacity(rows.len()); + // Populate new_ids with ALL symbol IDs for this project (for search filtering) + for (sym_id, _, _, _, _) in &rows { + new_ids.insert(*sym_id); + } for (sym_id, vec) in &embedded { vi.insert(*sym_id, vec).context("insert symbol embedding")?; - new_ids.insert(*sym_id); count += 1; } let insert_ms = t_insert.elapsed().as_millis(); tracing::debug!( - "embed_compute={}ms, usearch_insert={}ms, symbols={}, dims={}, provider={}", + "embed_compute={}ms, usearch_insert={}ms, re-embedded={}, total={}, skipped={}, dims={}, provider={}", compute_ms, insert_ms, count, + total_symbols, + skipped, active, active_provider_name() ); @@ -202,26 +251,36 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { vi.save().context("save vector index")?; } + // ── Update fingerprints for embedded symbols ───────────────────── + let mut update_fp = conn.prepare("UPDATE symbols SET embed_fingerprint = ?2 WHERE id = ?1")?; + for (sym_id, text) in &changed { + let mut hasher = DefaultHasher::new(); + text.hash(&mut hasher); + let fp = format!("{:016x}", hasher.finish()); + update_fp.execute(rusqlite::params![sym_id, fp])?; + } + // Cache project → symbol IDs for fast search-time filtering PROJECT_ID_CACHE .write() .unwrap() .insert(project_id, new_ids); - let tier = if cfg!(feature = "pretrained-embed") { + // Determine tier label + let provider = active_provider_name(); + let tier = if provider.contains("pretrained") { "pretrained" } else { "static" }; conn.execute( "UPDATE projects SET embedding_tier = ?3, embedding_dims = ?1, \ - last_embedded_at = datetime('now') WHERE id = ?2", - rusqlite::params![active, project_id, tier], + embedding_provider = ?4, last_embedded_at = datetime('now') WHERE id = ?2", + rusqlite::params![active, project_id, tier, provider], )?; tracing::info!( - "Embedded {count} symbols for project {project_id} (provider={}, dims={active})", - active_provider_name() + "Embedded {count}/{total_symbols} symbols for project {project_id} ({skipped} unchanged, provider={provider}, dims={active})", ); Ok(count) } diff --git a/src/index/schema.rs b/src/index/schema.rs index b7b716e..c5fb847 100644 --- a/src/index/schema.rs +++ b/src/index/schema.rs @@ -4,7 +4,7 @@ use rusqlite::Connection; /// Current schema version. #[allow(dead_code)] -const SCHEMA_VERSION: i32 = 6; +const SCHEMA_VERSION: i32 = 7; /// Run database migrations (creates tables if not exist). pub fn run_migrations(conn: &Connection) -> anyhow::Result<()> { @@ -40,6 +40,9 @@ pub fn run_migrations(conn: &Connection) -> anyhow::Result<()> { if current < 6 { migrate_v6(conn)?; } + if current < 7 { + migrate_v7(conn)?; + } Ok(()) } @@ -346,6 +349,45 @@ fn migrate_v6(conn: &Connection) -> anyhow::Result<()> { Ok(()) } +/// Migration v7: Add `embed_fingerprint` column to symbols for incremental re-embedding. +/// +/// Stores a hash of `name + signature` per symbol. On re-index, only symbols +/// whose fingerprint has changed need to be re-embedded — dramatically reducing +/// embedding time when a single file is modified. +fn migrate_v7(conn: &Connection) -> anyhow::Result<()> { + // SQLite ALTER TABLE ADD COLUMN is idempotent-safe with IF NOT EXISTS? No — + // SQLite doesn't support IF NOT EXISTS for ADD COLUMN. Use pragma check instead. + let cols: Vec = conn + .prepare("PRAGMA table_info(symbols)")? + .query_map([], |row| row.get::<_, String>(1))? // column 1 = name + .filter_map(|r| r.ok()) + .collect(); + + if !cols.iter().any(|c| c == "embed_fingerprint") { + conn.execute_batch("ALTER TABLE symbols ADD COLUMN embed_fingerprint TEXT;")?; + } + + // Also add to projects table: track which embedding backend was used. + // This allows detecting dimension mismatch when switching backends. + let pcols: Vec = conn + .prepare("PRAGMA table_info(projects)")? + .query_map([], |row| row.get::<_, String>(1))? + .filter_map(|r| r.ok()) + .collect(); + + if !pcols.iter().any(|c| c == "embedding_provider") { + conn.execute_batch("ALTER TABLE projects ADD COLUMN embedding_provider TEXT;")?; + } + + if !pcols.iter().any(|c| c == "embedding_dims") { + conn.execute_batch("ALTER TABLE projects ADD COLUMN embedding_dims INTEGER;")?; + } + + conn.execute("INSERT INTO schema_version (version) VALUES (7)", [])?; + + Ok(()) +} + /// Compute a stable hash of the indexing-relevant config. /// /// Any change to these fields will invalidate all stored fingerprints, @@ -644,7 +686,7 @@ mod tests { }) .unwrap(); assert_eq!(version, SCHEMA_VERSION); - assert_eq!(version, 6); + assert_eq!(version, 7); } #[test] diff --git a/src/main.rs b/src/main.rs index 94c1c60..74c2bc2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -738,8 +738,8 @@ async fn main() -> Result<()> { } } } else { - // Load config for config-hash invalidation - let skip_patterns = crate::config::loader::load_config( + // Load config for config-hash invalidation + brain embedding backend + let config = crate::config::loader::load_config( cli.global.config.as_deref(), None, None, @@ -747,8 +747,17 @@ async fn main() -> Result<()> { None, false, ) - .ok() - .map(|c| c.rules_config.index_skip_files); + .ok(); + let skip_patterns = config + .as_ref() + .map(|c| c.rules_config.index_skip_files.clone()); + + // Resolve embedding backend from brain config + let brain_mode = config + .as_ref() + .map(|c| c.brain.embedding.to_string()) + .unwrap_or_else(|| "auto".to_string()); + crate::embed::resolve_backend(&brain_mode); eprintln!("{}", "🔍 Indexing project...".cyan()); let stats = index::index_project_with_skip( @@ -1110,6 +1119,20 @@ async fn main() -> Result<()> { let conn = index::open_global_index()?; let project_id = index::ensure_project(&conn, &project_root)?; + // Resolve embedding backend from config for query embedding + let brain_mode = crate::config::loader::load_config( + cli.global.config.as_deref(), + None, + None, + None, + None, + false, + ) + .ok() + .map(|c| c.brain.embedding.to_string()) + .unwrap_or_else(|| "auto".to_string()); + crate::embed::resolve_backend(&brain_mode); + let results = index::brain::brain_search(&conn, project_id, &query_str, limit)?; if json {