From 9a6fd29a17a2ea12a7c7629f9914d2d806c675d4 Mon Sep 17 00:00:00 2001 From: Andrei Hasna Date: Tue, 23 Jun 2026 08:22:45 +0300 Subject: [PATCH] Fix legacy note search retrieval --- README.md | 21 +++-- docs/architecture/hybrid-semantic-search.md | 28 ++++--- src/cli.ts | 7 ++ src/retrieval.ts | 1 + src/search.ts | 82 ++++++++++++++++++- src/service.ts | 11 ++- tests/cli.test.ts | 90 +++++++++++++++++++++ tests/retrieval.test.ts | 36 +++++++++ tests/search.test.ts | 50 ++++++++++++ 9 files changed, 306 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7caa34c..2607a71 100644 --- a/README.md +++ b/README.md @@ -225,8 +225,8 @@ knowledge list|ls [options] ``` List compatibility JSON-store items with pagination, search, and tag filtering. Use `knowledge inventory` or `knowledge search` when an agent needs the -SQLite catalog, source chunks, generated wiki pages, artifacts, runs, and sync -state too. +SQLite catalog, source chunks, generated wiki pages, artifacts, runs, sync +state, or keyword retrieval across active compatibility notes too. | Flag | Description | |------|-------------| @@ -568,17 +568,24 @@ hash, updates permission/path/delete metadata, and records a local run ledger. Outbox inputs can be local files or allowed S3 objects, but raw source files remain owned by `open-files`. +Compatibility notes created by `knowledge add` live in the JSON item store. +They are returned by `knowledge search` and `knowledge search --context` through +keyword matching, but they are not chunked or embedded by `reindex`; reindexing +only refreshes SQLite source/wiki chunks and vector rows. + ### search ```bash knowledge search [--scope project] [--limit ] [--json] knowledge search --semantic [--model openai:text-embedding-3-small] [--scope project] [--json] knowledge search --context [--semantic] [--scope project] [--json] ``` -Run hybrid search over `chunks_fts`, generated wiki chunks, wiki/index catalog -rows, and optional vector results. The default path is local-only keyword and -catalog search. `--semantic` embeds the query and merges vector results from -`vector_index_entries`, preserving source refs, artifact URIs, citations, -revision/hash metadata, and provenance in each structured result. +Run hybrid search over active JSON-store notes, `chunks_fts`, generated wiki +chunks, wiki/index catalog rows, and optional vector results. The default path +is local-only keyword and catalog search. `--semantic` embeds the query and +merges vector results from `vector_index_entries`, preserving source refs, +artifact URIs, citations, revision/hash metadata, and provenance in each +structured result. JSON notes are keyword-only results with `kind: +legacy_item` and `knowledge://item/` source refs. `--context` returns a reranked context pack for agents: selected excerpts, assembled citations, freshness and permission notes, graph evidence from diff --git a/docs/architecture/hybrid-semantic-search.md b/docs/architecture/hybrid-semantic-search.md index ac50be0..f4d7896 100644 --- a/docs/architecture/hybrid-semantic-search.md +++ b/docs/architecture/hybrid-semantic-search.md @@ -40,6 +40,10 @@ Local mode starts with SQLite: - `wiki_pages`, `wiki_backlinks`, and `citations` provide graph and provenance signals. - `knowledge_indexes` tracks generated machine-readable shards. +- The compatibility JSON item store remains the source of truth for notes + created by `knowledge add`. Hybrid search reads active notes directly as + keyword-only `legacy_item` results; notes are not copied into `chunks` or + vector tables by default. The JSON vector representation is intentionally simple for the first local implementation. The retrieval interface should hide it so a later vector @@ -83,17 +87,19 @@ unauthorized content. 1. Normalize the query. 2. Embed the query if a semantic-capable provider is configured. 3. Run keyword FTS over source chunks and generated wiki chunks. -4. Search wiki page and machine-readable index catalog rows. -5. Run vector search over source chunks and wiki pages when semantic mode is +4. Keyword-match active compatibility JSON notes. +5. Search wiki page and machine-readable index catalog rows. +6. Run vector search over source chunks and wiki pages when semantic mode is requested. -6. Expand candidate pages through backlinks and citations. -7. Drop stale candidates whose source revision/hash no longer matches +7. Expand candidate pages through backlinks and citations. +8. Drop stale candidates whose source revision/hash no longer matches `open-files`. -8. Apply permission filters. -9. Merge and dedupe by source revision, wiki page, citation, and text hash. -10. Rerank by relevance, exact-match score, semantic score, freshness, citation +9. Apply permission filters. +10. Merge and dedupe by source revision, wiki page, citation, note id, and text + hash. +11. Rerank by relevance, exact-match score, semantic score, freshness, citation quality, and wiki authority. -11. Return structured results with source refs, citation spans, page refs, +12. Return structured results with source refs, citation spans, page refs, scores, and reason codes. ## Result Shape @@ -168,6 +174,9 @@ Reindexing is driven by source revisions: vector counts; `reindex enqueue` records missing work in `reindex_queue`. - `reindex embeddings` performs incremental refreshes, while `--full` clears and rebuilds `chunk_embeddings` and `vector_index_entries`. +- Compatibility JSON notes are not reindex work items. They remain searchable + through direct keyword matching and appear in context packs as + `legacy_item` excerpts with `knowledge://item/` refs. - Wiki pages should track the source revisions they cite so lint can flag stale pages. - Embedding refresh jobs should be idempotent and checkpointed in `runs` and @@ -177,7 +186,8 @@ Reindexing is driven by source revisions: - Local search works without network access for keyword-only retrieval. - Semantic search is optional and provider-gated. -- Every returned excerpt can resolve to a source ref or wiki page. +- Every returned excerpt can resolve to a source ref, wiki page, or + compatibility `knowledge://item/` note ref. - Permission filters run before model context assembly. - Retrieval internals can swap from JSON vectors to pgvector or managed vector stores without changing CLI/MCP result contracts. diff --git a/src/cli.ts b/src/cli.ts index 34f48e2..e6427dd 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -476,6 +476,7 @@ async function run(argv: string[]): Promise { if (!command || flags.help || command === 'help') { printCommandHelp(positional[1]); return; } const service = createKnowledgeService({ scope: flags.scope }); + const storePathOverridden = Boolean(flags.store); let storePath = flags.store; if (!storePath) { if (flags.scope === 'project' || flags.scope === 'local') { @@ -484,6 +485,9 @@ async function run(argv: string[]): Promise { storePath = defaultStorePath(); } } + if (!storePathOverridden && (command === 'search' || command === 'ask' || command === 'build')) { + ensureStore(storePath); + } if (command === 'inventory') { const inventory = service.inventory({ @@ -1131,6 +1135,7 @@ async function run(argv: string[]): Promise { modelRef: flags.model, dimensions: flags.dimensions, fake: flags.fake, + legacyStorePath: storePath, }); output({ ok: true, ...context, message: `${context.excerpts.length} context excerpt(s)` }, flags.json); return; @@ -1142,6 +1147,7 @@ async function run(argv: string[]): Promise { modelRef: flags.model, dimensions: flags.dimensions, fake: flags.fake, + legacyStorePath: storePath, }); output({ ok: true, ...result, message: `${result.results.length} search result(s)` }, flags.json); return; @@ -1177,6 +1183,7 @@ async function run(argv: string[]): Promise { fake: flags.fake, generate: flags.generate, approveWrite: flags.approveWrite, + legacyStorePath: storePath, }); output({ ok: true, ...result, message: result.generated ? 'Generated answer with citations' : 'Prepared citation context draft' }, flags.json); return; diff --git a/src/retrieval.ts b/src/retrieval.ts index 54f052f..5cebe95 100644 --- a/src/retrieval.ts +++ b/src/retrieval.ts @@ -149,6 +149,7 @@ function citationScore(result: HybridSearchEntry): number { function authorityScore(result: HybridSearchEntry): number { if (result.kind === 'wiki_chunk') return 0.85; if (result.kind === 'source_chunk') return 0.8; + if (result.kind === 'legacy_item') return 0.6; if (result.kind === 'wiki_page') return 0.65; return 0.55; } diff --git a/src/search.ts b/src/search.ts index 5942e5e..b7e33c6 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,14 +1,17 @@ import type { Database } from 'bun:sqlite'; +import { existsSync, readFileSync } from 'node:fs'; import { migrateKnowledgeDb, openKnowledgeDb } from './knowledge-db'; import { searchVectorIndex, type EmbeddingRuntimeOptions } from './embeddings'; import { sourceProvenance, type GeneratedArtifactProvenance, type KnowledgeProvenance } from './provenance'; +import type { KnowledgeItem } from './store'; import type { KnowledgeConfig } from './workspace'; -export type SearchResultKind = 'source_chunk' | 'wiki_chunk' | 'wiki_page' | 'knowledge_index'; +export type SearchResultKind = 'source_chunk' | 'wiki_chunk' | 'legacy_item' | 'wiki_page' | 'knowledge_index'; export type SearchProvenance = KnowledgeProvenance | GeneratedArtifactProvenance; export interface HybridSearchOptions extends EmbeddingRuntimeOptions { dbPath: string; + legacyStorePath?: string; query: string; limit?: number; semantic?: boolean; @@ -281,6 +284,50 @@ function selectKnowledgeIndexes(db: Database, terms: string[], limit: number): I ).all(...likeParams(terms, fields.length), limit); } +function readLegacyItems(path?: string): KnowledgeItem[] { + if (!path || !existsSync(path)) return []; + try { + const parsed = JSON.parse(readFileSync(path, 'utf8')) as { items?: unknown }; + if (!parsed || !Array.isArray(parsed.items)) return []; + return parsed.items.filter((item): item is KnowledgeItem => { + return Boolean( + item + && typeof item === 'object' + && typeof (item as KnowledgeItem).id === 'string' + && typeof (item as KnowledgeItem).title === 'string' + && typeof (item as KnowledgeItem).content === 'string', + ); + }); + } catch { + return []; + } +} + +function legacyItemHaystack(item: KnowledgeItem): string { + return [ + item.id, + item.short_id, + item.title, + item.content, + item.url, + ...(item.tags ?? []), + ].filter((value): value is string => typeof value === 'string' && value.length > 0).join(' ').toLowerCase(); +} + +function selectLegacyItems(path: string | undefined, terms: string[], limit: number): Array<{ + item: KnowledgeItem; + score: number; +}> { + if (terms.length === 0) return []; + return readLegacyItems(path) + .filter((item) => item.archived !== true) + .map((item) => ({ item, haystack: legacyItemHaystack(item) })) + .filter(({ haystack }) => terms.some((term) => haystack.includes(term))) + .map(({ item, haystack }) => ({ item, score: catalogScore(haystack, terms) })) + .sort((a, b) => b.score - a.score || a.item.id.localeCompare(b.item.id)) + .slice(0, limit); +} + function chunkResult(row: FtsChunkRow, keywordScore: number): HybridSearchEntry { const metadata = parseJsonObject(row.chunk_metadata_json); const provenance = provenanceForChunk(row); @@ -319,6 +366,31 @@ function chunkResult(row: FtsChunkRow, keywordScore: number): HybridSearchEntry return result; } +function legacyItemResult(item: KnowledgeItem, keywordScore: number): HybridSearchEntry { + const uri = `knowledge://item/${encodeURIComponent(item.id)}`; + const result: HybridSearchEntry = { + kind: 'legacy_item', + id: item.id, + title: item.title, + text: item.content, + score: 0, + scores: { keyword: keywordScore }, + source: { + uri, + ref: uri, + kind: 'legacy_item', + revision: null, + hash: null, + }, + citation: null, + artifact: null, + provenance: null, + reasons: ['legacy_note_match', 'keyword_match'], + }; + result.score = combinedScore(result.scores, result.citation); + return result; +} + function wikiPageResult(row: WikiPageRow, terms: string[]): HybridSearchEntry { const metadata = parseJsonObject(row.metadata_json); const score = catalogScore(`${row.path} ${row.title} ${row.artifact_uri ?? ''} ${row.metadata_json}`.toLowerCase(), terms); @@ -395,8 +467,9 @@ function sortResults(results: HybridSearchEntry[]): HybridSearchEntry[] { const kindOrder: Record = { source_chunk: 0, wiki_chunk: 1, - wiki_page: 2, - knowledge_index: 3, + legacy_item: 2, + wiki_page: 3, + knowledge_index: 4, }; return results.sort((a, b) => { if (b.score !== a.score) return b.score - a.score; @@ -429,7 +502,10 @@ export async function hybridSearch(options: HybridSearchOptions): Promise mergeResult(merged, legacyItemResult(item, score))); wikiRows.forEach((row) => mergeResult(merged, wikiPageResult(row, terms))); indexRows.forEach((row) => mergeResult(merged, indexResult(row, terms))); } finally { diff --git a/src/service.ts b/src/service.ts index e591ea8..800981a 100644 --- a/src/service.ts +++ b/src/service.ts @@ -75,7 +75,7 @@ import { type StorageContract, type StorageValidationResult, } from './storage-contract'; -import type { KnowledgeItem } from './store'; +import { ensureStore, type KnowledgeItem } from './store'; import { initializeWikiLayout, recordWikiLayoutCatalog } from './wiki-layout'; import { canonicalHasnaXyzKnowledgeStorage, @@ -1700,27 +1700,36 @@ export class KnowledgeService { async search(options: Omit) { const workspace = this.ensureWorkspace(); + const legacyStorePath = options.legacyStorePath ?? workspace.jsonStorePath; + if (!options.legacyStorePath) ensureStore(legacyStorePath); return hybridSearch({ ...options, dbPath: workspace.knowledgeDbPath, + legacyStorePath, config: this.config(), }); } async retrieveContext(options: Omit) { const workspace = this.ensureWorkspace(); + const legacyStorePath = options.legacyStorePath ?? workspace.jsonStorePath; + if (!options.legacyStorePath) ensureStore(legacyStorePath); return retrieveKnowledgeContext({ ...options, dbPath: workspace.knowledgeDbPath, + legacyStorePath, config: this.config(), }); } async runPrompt(options: Omit) { const workspace = this.ensureWorkspace(); + const legacyStorePath = options.legacyStorePath ?? workspace.jsonStorePath; + if (!options.legacyStorePath) ensureStore(legacyStorePath); return runKnowledgePrompt({ ...options, dbPath: workspace.knowledgeDbPath, + legacyStorePath, config: this.config(), }); } diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 04e8722..3a8c1d5 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -295,6 +295,96 @@ describe('knowledge cli', () => { expect(updateOut.item.content).toBe('Updated body'); }); + test('global notes added through CLI are searchable and available as context', () => { + const home = mkdtempSync(join(tmpdir(), 'ok-global-note-search-home-')); + const env = { + HOME: home, + HASNA_KNOWLEDGE_AUTH_DIR: join(home, 'auth'), + }; + + const add = runCli([ + 'add', + 'Hasna OSS boundary', + 'local-first hosted wrapper open actions guardrails open orgs', + '--scope', + 'global', + '--json', + ], undefined, env); + expect(add.exitCode).toBe(0); + const addOut = JSON.parse(new TextDecoder().decode(add.stdout)); + + const update = runCli(['update', '--id', addOut.item.id, '--tag', 'opensource', '--scope', 'global', '--json'], undefined, env); + expect(update.exitCode).toBe(0); + + const list = runCli(['list', '--tag', 'opensource', '--scope', 'global', '--json'], undefined, env); + expect(list.exitCode).toBe(0); + expect(JSON.parse(new TextDecoder().decode(list.stdout)).total).toBe(1); + + const inventory = runCli(['inventory', '--scope', 'global', '--json'], undefined, env); + expect(inventory.exitCode).toBe(0); + const inventoryOut = JSON.parse(new TextDecoder().decode(inventory.stdout)); + expect(inventoryOut.summary.legacy_items).toBe(1); + expect(inventoryOut.summary.chunks).toBe(0); + + const search = runCli(['search', 'Hasna OSS boundary', '--scope', 'global', '--json'], undefined, env); + expect(search.exitCode).toBe(0); + const searchOut = JSON.parse(new TextDecoder().decode(search.stdout)); + expect(searchOut.counts.keyword_results).toBe(1); + expect(searchOut.results[0]).toMatchObject({ + kind: 'legacy_item', + id: addOut.item.id, + title: 'Hasna OSS boundary', + source: { uri: `knowledge://item/${addOut.item.id}` }, + }); + + const context = runCli([ + 'search', + 'local-first hosted wrapper open actions guardrails open orgs', + '--context', + '--scope', + 'global', + '--json', + ], undefined, env); + expect(context.exitCode).toBe(0); + const contextOut = JSON.parse(new TextDecoder().decode(context.stdout)); + expect(contextOut.results[0]).toMatchObject({ kind: 'legacy_item', id: addOut.item.id }); + expect(contextOut.excerpts[0].text).toContain('local-first hosted wrapper'); + + const reindex = runCli(['reindex', 'enqueue', '--scope', 'global', '--json'], undefined, env); + expect(reindex.exitCode).toBe(0); + expect(JSON.parse(new TextDecoder().decode(reindex.stdout)).enqueued).toBe(0); + }); + + test('global search migrates the old legacy note store before list is run', () => { + const home = mkdtempSync(join(tmpdir(), 'ok-global-note-legacy-migrate-')); + const legacyDir = join(home, '.open-knowledge'); + mkdirSync(legacyDir, { recursive: true }); + writeFileSync(join(legacyDir, 'db.json'), JSON.stringify({ + items: [{ + id: 'k_legacy_global_search', + title: 'Legacy Global Search', + content: 'tokenxyz first command migration path', + url: null, + tags: ['legacy'], + created_at: '2026-06-23T00:00:00.000Z', + updated_at: '2026-06-23T00:01:00.000Z', + }], + })); + const env = { + HOME: home, + HASNA_KNOWLEDGE_AUTH_DIR: join(home, 'auth'), + }; + + const search = runCli(['search', 'tokenxyz', '--scope', 'global', '--json'], undefined, env); + expect(search.exitCode).toBe(0); + const searchOut = JSON.parse(new TextDecoder().decode(search.stdout)); + expect(searchOut.results[0]).toMatchObject({ + kind: 'legacy_item', + id: 'k_legacy_global_search', + }); + expect(existsSync(join(home, '.hasna', 'apps', 'knowledge', 'db.json'))).toBe(true); + }); + test('project scope uses .hasna/apps/knowledge workspace', () => { const dir = mkdtempSync(join(tmpdir(), 'ok-workspace-')); diff --git a/tests/retrieval.test.ts b/tests/retrieval.test.ts index 14511eb..a3bbfb1 100644 --- a/tests/retrieval.test.ts +++ b/tests/retrieval.test.ts @@ -7,6 +7,42 @@ import { retrieveKnowledgeContext } from '../src/retrieval'; import { ingestSourceRef } from '../src/source-ingest'; describe('knowledge retrieval context packs', () => { + test('assembles context excerpts from legacy JSON notes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'ok-retrieval-legacy-note-')); + const dbPath = join(dir, 'knowledge.db'); + const legacyStorePath = join(dir, 'db.json'); + writeFileSync(legacyStorePath, JSON.stringify({ + items: [{ + id: 'k_note_context', + title: 'Hasna OSS boundary', + content: 'local-first hosted wrapper open actions guardrails open orgs', + url: null, + tags: ['opensource'], + created_at: '2026-06-23T00:00:00.000Z', + updated_at: '2026-06-23T00:01:00.000Z', + }], + })); + + const context = await retrieveKnowledgeContext({ + dbPath, + legacyStorePath, + query: 'local-first hosted wrapper open actions guardrails open orgs', + limit: 5, + }); + + expect(context.results[0]).toMatchObject({ + kind: 'legacy_item', + id: 'k_note_context', + source: { ref: 'knowledge://item/k_note_context' }, + }); + expect(context.citations[0]).toMatchObject({ + kind: 'legacy_item', + source_uri: 'knowledge://item/k_note_context', + chunk_id: null, + }); + expect(context.excerpts[0].text).toContain('local-first hosted wrapper'); + }); + test('reranks search results and assembles citations, excerpts, and graph evidence', async () => { const dir = mkdtempSync(join(tmpdir(), 'ok-retrieval-')); const dbPath = join(dir, 'knowledge.db'); diff --git a/tests/search.test.ts b/tests/search.test.ts index e083af8..4436b5b 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -10,6 +10,56 @@ import { ingestSourceRef } from '../src/source-ingest'; import { initializeWikiLayout, recordWikiLayoutCatalog } from '../src/wiki-layout'; describe('hybrid knowledge search', () => { + test('searches active legacy JSON notes as keyword results', async () => { + const dir = mkdtempSync(join(tmpdir(), 'ok-hybrid-legacy-notes-')); + const dbPath = join(dir, 'knowledge.db'); + const legacyStorePath = join(dir, 'db.json'); + writeFileSync(legacyStorePath, JSON.stringify({ + items: [ + { + id: 'k_note_boundary', + title: 'Hasna OSS boundary', + content: 'local-first hosted wrapper open actions guardrails open orgs', + url: null, + tags: ['opensource'], + created_at: '2026-06-23T00:00:00.000Z', + updated_at: '2026-06-23T00:01:00.000Z', + }, + { + id: 'k_archived_note', + title: 'Archived boundary', + content: 'This archived note should not be returned.', + url: null, + tags: ['opensource'], + archived: true, + created_at: '2026-06-23T00:00:00.000Z', + updated_at: '2026-06-23T00:01:00.000Z', + }, + ], + })); + + const results = await hybridSearch({ + dbPath, + legacyStorePath, + query: 'Hasna OSS boundary', + limit: 5, + }); + + expect(results.counts.keyword_results).toBe(1); + expect(results.results).toHaveLength(1); + expect(results.results[0]).toMatchObject({ + kind: 'legacy_item', + id: 'k_note_boundary', + title: 'Hasna OSS boundary', + text: 'local-first hosted wrapper open actions guardrails open orgs', + source: { + uri: 'knowledge://item/k_note_boundary', + kind: 'legacy_item', + }, + reasons: ['legacy_note_match', 'keyword_match'], + }); + }); + test('searches source chunks, wiki chunks, catalog rows, and optional vectors', async () => { const dir = mkdtempSync(join(tmpdir(), 'ok-hybrid-search-')); const dbPath = join(dir, 'knowledge.db');