Skip to content

feat(search): AND semantics, field boosting, recency + graph distance ranking #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tstapler merged 1 commit into from
Apr 23, 2026
Merged

feat(search): AND semantics, field boosting, recency + graph distance ranking #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions kmp/src/commonMain/kotlin/dev/stapler/stelekit/repository/GraphRepository.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,29 +472,46 @@ data class DateRange(
val endDate: kotlin.time.Instant? = null
)

/** A page result with an optional snippet from FTS5 highlight(). */
/** A page result with an optional snippet and raw BM25 score from FTS5. */
data class SearchedPage(
val page: Page,
val snippet: String? = null
val snippet: String? = null,
val bm25Score: Double = 0.0
)

/** A block result with an optional snippet from FTS5 highlight(). */
/** A block result with an optional snippet and raw BM25 score from FTS5. */
data class SearchedBlock(
val block: Block,
val snippet: String? = null
val snippet: String? = null,
val bm25Score: Double = 0.0
)

/**
* A search hit carrying an absolute relevance score suitable for cross-type ranking.
*
* BM25 returns negative values (more negative = more relevant). [score] is the
* absolute value, optionally multiplied by [SqlDelightSearchRepository.PAGE_BOOST]
* for page-title hits so they sort above body-text hits.
*/
sealed class RankedSearchHit {
abstract val score: Double
data class PageHit(val page: Page, val snippet: String?, override val score: Double) : RankedSearchHit()
data class BlockHit(val block: Block, val snippet: String?, override val score: Double) : RankedSearchHit()
}

/**
* Search result with metadata.
*
* [searchedPages] and [searchedBlocks] carry BM25-ranked results with highlight snippets.
* [blocks] / [pages] are kept for backward compatibility with callers that do not need snippets.
* [ranked] interleaves pages and blocks sorted by boosted relevance score (highest first).
*/
data class SearchResult(
val blocks: List<Block>,
val pages: List<Page>,
val searchedBlocks: List<SearchedBlock> = emptyList(),
val searchedPages: List<SearchedPage> = emptyList(),
val ranked: List<RankedSearchHit> = emptyList(),
val totalCount: Int,
val hasMore: Boolean
)
Expand Down
114 changes: 103 additions & 11 deletions kmp/src/commonMain/kotlin/dev/stapler/stelekit/repository/SqlDelightSearchRepository.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@ import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.flowOn
import kotlin.time.Instant
import kotlin.time.Duration.Companion.milliseconds

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Duration.Companion.milliseconds is imported but never used in this file. Please remove it to avoid unused-import warnings and keep imports tidy.

Suggested change
import kotlin.time.Duration.Companion.milliseconds

Copilot uses AI. Check for mistakes.
import kotlin.math.abs
import kotlin.math.exp
import kotlin.Result.Companion.success

/**
* SQLDelight implementation of SearchRepository.
*
* Block content is searched via FTS5 with BM25 ranking.
* Page names are searched via FTS5 (pages_fts) with BM25 ranking.
* Queries are built by [FtsQueryBuilder] which handles phrase search and multi-token OR.
* Queries are built by [FtsQueryBuilder] which handles phrase search and multi-token AND (with OR fallback).
* Page-title hits are boosted by [PAGE_BOOST] relative to block-content hits in [ranked] results.
*/
class SqlDelightSearchRepository(
private val database: SteleDatabase,
Expand All @@ -34,6 +38,15 @@ class SqlDelightSearchRepository(
private val queries = database.steleDatabaseQueries
private val spanEmitter = SpanEmitter(ringBuffer)

companion object {
/** Page-title hits are multiplied by this factor before ranking against block hits. */
const val PAGE_BOOST = 5.0
/** Results on a page directly linked to/from the current page get this multiplier. */
const val GRAPH_BOOST = 3.0
/** Recency half-life in days: a result edited this many days ago gets half the recency bonus. */
const val RECENCY_HALFLIFE_DAYS = 14.0
}

override fun searchBlocksByContent(query: String, limit: Int, offset: Int): Flow<Result<List<Block>>> = flow {
try {
val ftsQuery = FtsQueryBuilder.build(query)
Expand All @@ -43,11 +56,22 @@ class SqlDelightSearchRepository(
val spanId = UuidGenerator.generateV7()
CurrentSpanContext.set(ActiveSpanContext(traceId, spanId))
val results = try {
queries.searchBlocksByContentFts(
val andResults = queries.searchBlocksByContentFts(
query = ftsQuery,
limit = limit.toLong(),
offset = offset.toLong()
).executeAsList().map { it.toBlockModel() }
).executeAsList()
if (andResults.isNotEmpty()) {
andResults.map { it.toBlockModel() }
} else {
val orQuery = FtsQueryBuilder.buildOr(query)
if (orQuery.isEmpty()) emptyList()
else queries.searchBlocksByContentFts(
query = orQuery,
limit = limit.toLong(),
offset = offset.toLong()
).executeAsList().map { it.toBlockModel() }
}
} finally {
CurrentSpanContext.set(null)
}
Expand Down Expand Up @@ -150,13 +174,25 @@ class SqlDelightSearchRepository(
DataType.TITLES in dataTypes
) {
try {
queries.searchPagesByNameFts(
val andPages = queries.searchPagesByNameFts(
query = ftsQuery,
limit = searchRequest.limit.toLong()
).executeAsList().map { row ->
).executeAsList()
val pageRows = if (andPages.isNotEmpty()) {
andPages
} else {
val orQuery = FtsQueryBuilder.buildOr(rawQuery)
if (orQuery.isEmpty()) emptyList()
else queries.searchPagesByNameFts(
query = orQuery,
limit = searchRequest.limit.toLong()
).executeAsList()
}
pageRows.map { row ->
SearchedPage(
page = row.toPageModel(),
snippet = row.highlight?.takeIf { it.isNotBlank() }
snippet = row.highlight?.takeIf { it.isNotBlank() },
bm25Score = row.bm25_score
)
}.applyPageScope(scope, searchRequest.pageUuid)
} catch (_: Exception) {
Expand Down Expand Up @@ -187,20 +223,34 @@ class SqlDelightSearchRepository(
).executeAsList().map { row ->
SearchedBlock(
block = row.toBlockModel(),
snippet = row.highlight?.takeIf { it.isNotBlank() }
snippet = row.highlight?.takeIf { it.isNotBlank() },
bm25Score = row.bm25_score
Comment on lines 223 to +227

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For SearchScope.CURRENT_PAGE, this code path executes only the AND-built ftsQuery via searchBlocksByContentFtsInPage(...) and does not apply the OR fallback used in the other search paths when the AND query returns zero rows. This can still produce empty results for multi-term queries in current-page scope.

Copilot uses AI. Check for mistakes.
)
}
} else emptyList()
}
else -> {
queries.searchBlocksByContentFts(
val andBlocks = queries.searchBlocksByContentFts(
query = ftsQuery,
limit = searchRequest.limit.toLong(),
offset = searchRequest.offset.toLong()
).executeAsList().map { row ->
).executeAsList()
val blockRows = if (andBlocks.isNotEmpty()) {
andBlocks
} else {
val orQuery = FtsQueryBuilder.buildOr(rawQuery)
if (orQuery.isEmpty()) emptyList()
else queries.searchBlocksByContentFts(
query = orQuery,
limit = searchRequest.limit.toLong(),
offset = searchRequest.offset.toLong()
).executeAsList()
}
blockRows.map { row ->
SearchedBlock(
block = row.toBlockModel(),
snippet = row.highlight?.takeIf { it.isNotBlank() }
snippet = row.highlight?.takeIf { it.isNotBlank() },
bm25Score = row.bm25_score
)
}.applyBlockScope(scope)
}
Expand All @@ -210,19 +260,61 @@ class SqlDelightSearchRepository(
}
} else emptyList()

val neighbourPageUuids = searchRequest.pageUuid
?.let { runCatching { queries.selectNeighbourPageUuids(it).executeAsList().toSet() }.getOrDefault(emptySet()) }
?: emptySet()
val nowMs = HistogramWriter.epochMs()
val ranked = buildRankedList(searchedPages, searchedBlocks, neighbourPageUuids, nowMs)
emit(success(SearchResult(
blocks = searchedBlocks.map { it.block },
pages = searchedPages.map { it.page },
searchedBlocks = searchedBlocks,
searchedPages = searchedPages,
totalCount = searchedBlocks.size + searchedPages.size,
ranked = ranked,
totalCount = ranked.size,
hasMore = false
)))
} catch (e: Exception) {
emit(Result.failure(e))
}
}.flowOn(PlatformDispatcher.DB)

// ── Ranking helpers ────────────────────────────────────────────────────

private fun buildRankedList(
pages: List<SearchedPage>,
blocks: List<SearchedBlock>,
neighbourPageUuids: Set<String>,
nowMs: Long,
): List<RankedSearchHit> {
val pageHits = pages.map { sp ->
val bm25 = abs(sp.bm25Score)
val score = bm25 * PAGE_BOOST *
recencyMultiplier(sp.page.updatedAt.toEpochMilliseconds(), nowMs) *
graphMultiplier(sp.page.uuid, neighbourPageUuids)
RankedSearchHit.PageHit(sp.page, sp.snippet, score)
Comment on lines +291 to +295

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

abs(sp.bm25Score) is not a monotonic transform of BM25 across all possible return values. Since the SQL queries rank by ORDER BY bm25(...) (smaller = more relevant), converting to a cross-type “higher is better” score should use a monotonic mapping (e.g., -sp.bm25Score). With the current abs, if bm25() ever returns positive values, worse matches can incorrectly get higher scores.

Copilot uses AI. Check for mistakes.
}
val blockHits = blocks.map { sb ->
val bm25 = abs(sb.bm25Score)
val score = bm25 *
recencyMultiplier(sb.block.updatedAt.toEpochMilliseconds(), nowMs) *
graphMultiplier(sb.block.pageUuid, neighbourPageUuids)
RankedSearchHit.BlockHit(sb.block, sb.snippet, score)
Comment on lines +298 to +302

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same scoring issue for blocks: abs(sb.bm25Score) can invert relevance ordering if BM25 is positive. Use a monotonic transform consistent with ORDER BY bm25(...) (e.g., -sb.bm25Score) so higher score always means more relevant.

Copilot uses AI. Check for mistakes.
}
return (pageHits + blockHits).sortedByDescending { it.score }
}

/** Returns 1.0 + exp(-daysSinceEdit / halfLife) — between ~2.0 (today) and ~1.0 (old). */
private fun recencyMultiplier(updatedAtMs: Long, nowMs: Long): Double {
if (updatedAtMs <= 0) return 1.0
val daysSince = (nowMs - updatedAtMs).coerceAtLeast(0L) / 86_400_000.0
return 1.0 + exp(-daysSince / RECENCY_HALFLIFE_DAYS)
Comment on lines +307 to +311

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The KDoc and constant name say “half-life”, but exp(-daysSince / RECENCY_HALFLIFE_DAYS) does not halve the bonus at RECENCY_HALFLIFE_DAYS (it yields ~0.367 of the bonus). If you want a true half-life, the exponent needs a ln(2) factor (or rename the constant/documentation to match the implemented decay).

Copilot uses AI. Check for mistakes.
}

/** Returns GRAPH_BOOST if the page is a 1-hop neighbour of the current page, else 1.0. */
private fun graphMultiplier(pageUuid: String, neighbourPageUuids: Set<String>): Double =
if (pageUuid in neighbourPageUuids) GRAPH_BOOST else 1.0

// ── Scope helpers ──────────────────────────────────────────────────────

private fun List<SearchedPage>.applyPageScope(
Expand Down
49 changes: 26 additions & 23 deletions kmp/src/commonMain/kotlin/dev/stapler/stelekit/search/FtsQueryBuilder.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ package dev.stapler.stelekit.search
* Features:
* - Balanced double-quote pairs are preserved as FTS5 phrase segments.
* - Unbalanced quotes are stripped and the content treated as plain tokens.
* - Multi-token queries use OR semantics so partial matches surface at lower rank.
* - A `*` prefix-match wildcard is appended to the last unquoted token, supporting
* search-as-you-type.
* - Multi-token queries use AND semantics — all terms must appear in a matching row.
* - A `*` prefix-match wildcard is appended to every plain token, supporting
* search-as-you-type and partial word matching.
* - Dangerous bare operators (AND/OR/NOT at the start) are stripped.
* - Characters that would break FTS5 syntax (`:`, `^`, `~`, `{`, `}`, `[`, `]`, `!`)
* are removed from plain tokens.
Expand All @@ -19,41 +19,44 @@ object FtsQueryBuilder {
private val LEADING_OPERATORS = setOf("AND", "OR", "NOT")

/**
* Converts a raw user query into a safe FTS5 MATCH expression.
* Converts a raw user query into a safe FTS5 MATCH expression using AND semantics.
* Every plain token receives a `*` prefix-match wildcard.
*
* Examples:
* "2025 Taxes" → "2025 OR taxes*"
* '"meeting notes"' → '"meeting notes"'
* '"meeting notes" tax' → '"meeting notes" OR tax*'
* '"unclosed' → 'unclosed*'
* 'OR AND taxes' → 'taxes*'
* '' → ''
* "2025 Taxes" → "2025* AND Taxes*"
* '"meeting notes"' → '"meeting notes"'
* '"meeting notes" tax' → '"meeting notes" AND tax*'
* '"unclosed' → 'unclosed*'
* 'OR AND taxes' → 'taxes*'
* '' → ''
*/
fun build(rawQuery: String): String {
fun build(rawQuery: String): String = buildQuery(rawQuery, " AND ")

/**
* Like [build] but joins tokens with OR semantics.
* Use as a fallback when an AND query returns no results.
*
* Examples:
* "2025 Taxes" → "2025* OR Taxes*"
*/
fun buildOr(rawQuery: String): String = buildQuery(rawQuery, " OR ")

private fun buildQuery(rawQuery: String, joinStr: String): String {
val trimmed = rawQuery.trim()
if (trimmed.isEmpty()) return ""

val segments = parseSegments(trimmed)
if (segments.isEmpty()) return ""

// Locate the last plain-token segment index for prefix wildcard
val lastTokenIdx = segments.indexOfLast { it is Segment.Token }

return segments.mapIndexed { i, seg ->
return segments.mapNotNull { seg ->
when (seg) {
is Segment.Phrase -> "\"${seg.content}\""
is Segment.Token -> {
val clean = seg.content.replace(STRIP_CHARS, "").trim()
when {
clean.isEmpty() -> null
i == lastTokenIdx -> "$clean*"
else -> clean
}
if (clean.isEmpty()) null else "$clean*"
}
}
}
.filterNotNull()
.joinToString(" OR ")
}.joinToString(joinStr)
}

// ── Internal parser ──────────────────────────────────────────────────────
Expand Down
24 changes: 21 additions & 3 deletions kmp/src/commonMain/sqldelight/dev/stapler/stelekit/db/SteleDatabase.sq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,21 @@ UPDATE pages SET is_favorite = ? WHERE uuid = ?;
deleteAllPages:
DELETE FROM pages;

-- All page UUIDs reachable in 1 hop from a given page (outgoing or incoming links).
-- Used to compute graph-distance boost during search ranking.
selectNeighbourPageUuids:
SELECT DISTINCT to_b.page_uuid AS page_uuid
FROM block_references br
JOIN blocks from_b ON from_b.uuid = br.from_block_uuid
JOIN blocks to_b ON to_b.uuid = br.to_block_uuid
WHERE from_b.page_uuid = :pageUuid AND to_b.page_uuid != :pageUuid
UNION
SELECT DISTINCT from_b.page_uuid AS page_uuid
Comment on lines +358 to +364

Copilot AI Apr 23, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

selectNeighbourPageUuids uses SELECT DISTINCT ... in each branch, but the branches are combined with UNION (which is distinct by default). The inner DISTINCT is redundant work; consider removing it (or using UNION ALL with a single outer SELECT DISTINCT).

Suggested change
SELECT DISTINCT to_b.page_uuid AS page_uuid
FROM block_references br
JOIN blocks from_b ON from_b.uuid = br.from_block_uuid
JOIN blocks to_b ON to_b.uuid = br.to_block_uuid
WHERE from_b.page_uuid = :pageUuid AND to_b.page_uuid != :pageUuid
UNION
SELECT DISTINCT from_b.page_uuid AS page_uuid
SELECT to_b.page_uuid AS page_uuid
FROM block_references br
JOIN blocks from_b ON from_b.uuid = br.from_block_uuid
JOIN blocks to_b ON to_b.uuid = br.to_block_uuid
WHERE from_b.page_uuid = :pageUuid AND to_b.page_uuid != :pageUuid
UNION
SELECT from_b.page_uuid AS page_uuid

Copilot uses AI. Check for mistakes.
FROM block_references br
JOIN blocks from_b ON from_b.uuid = br.from_block_uuid
JOIN blocks to_b ON to_b.uuid = br.to_block_uuid
WHERE to_b.page_uuid = :pageUuid AND from_b.page_uuid != :pageUuid;

-- Reference queries
selectOutgoingReferences:
SELECT b.* FROM blocks b
Expand Down Expand Up @@ -454,7 +469,8 @@ SELECT
b.updated_at,
b.properties,
b.version,
highlight(blocks_fts, 0, '<em>', '</em>') AS highlight
highlight(blocks_fts, 0, '<em>', '</em>') AS highlight,
bm25(blocks_fts) AS bm25_score
FROM blocks_fts bm
JOIN blocks b ON b.id = bm.rowid
WHERE blocks_fts MATCH :query
Expand All @@ -474,7 +490,8 @@ SELECT
b.updated_at,
b.properties,
b.version,
highlight(blocks_fts, 0, '<em>', '</em>') AS highlight
highlight(blocks_fts, 0, '<em>', '</em>') AS highlight,
bm25(blocks_fts) AS bm25_score
FROM blocks_fts bm
JOIN blocks b ON b.id = bm.rowid
WHERE blocks_fts MATCH :query
Expand Down Expand Up @@ -502,7 +519,8 @@ SELECT
p.is_journal,
p.journal_date,
p.is_content_loaded,
highlight(pages_fts, 0, '<em>', '</em>') AS highlight
highlight(pages_fts, 0, '<em>', '</em>') AS highlight,
bm25(pages_fts) AS bm25_score
FROM pages_fts pf
JOIN pages p ON p.rowid = pf.rowid
WHERE pages_fts MATCH :query
Expand Down
Loading
Loading