From c85666c3580658612478ab3bfb2986c811495358 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 7 Jul 2026 00:11:02 -0700 Subject: [PATCH] perf(selfhost): skip reporting-exporter's rebuild when the source is unchanged Every 30s cycle rebuilt the entire Grafana reporting SQLite mirror from scratch, regardless of whether the source data had actually changed -- measured at 91GB of cumulative block I/O in 37 hours on a real self-host instance, growing without bound as ai_usage_events (an append-only log of every AI call) accumulates. A live review pipeline is bursty: most cycles change nothing since the last export. Add a cheap COUNT+MAX fingerprint per source table (both the SQLite and Postgres source paths) and skip the full rebuild when it matches the last run's and a last-good output DB already exists. Fails open: any error computing the fingerprint falls through to the existing full-rebuild path unchanged, so this is purely an optimization, never a new failure mode. Closes #3895 --- scripts/export-grafana-reporting-db.sh | 67 +++++++++++++++++ test/unit/selfhost-grafana-reporting.test.ts | 79 +++++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/scripts/export-grafana-reporting-db.sh b/scripts/export-grafana-reporting-db.sh index b66f10bfe4..a8873dc83f 100644 --- a/scripts/export-grafana-reporting-db.sh +++ b/scripts/export-grafana-reporting-db.sh @@ -6,6 +6,7 @@ PG_DB="${GITTENSORY_REPORTING_SOURCE_DATABASE_URL:-${DATABASE_URL:-}}" OUT_DIR="${GITTENSORY_REPORTING_DIR:-/reporting}" OUT_DB="${GITTENSORY_REPORTING_DB:-$OUT_DIR/gittensory-reporting.sqlite}" TMP_DB="${OUT_DB}.tmp" +FINGERPRINT_FILE="${OUT_DB}.fingerprint" CSV_TMP_DIR="$(mktemp -d)" cleanup() { @@ -134,8 +135,72 @@ sqlite_import_csv() { SQL } +# Incremental fast-path (#3895): a live review pipeline is bursty -- most 30s cycles change nothing since the +# last export, yet the full rebuild below re-exports and re-imports every row of every table every time. That +# cost grows without bound as ai_usage_events (an append-only log of every AI call) accumulates -- measured on +# a real self-host instance at 91GB of cumulative block I/O in 37 hours. A cheap COUNT+MAX fingerprint per +# table is orders of magnitude cheaper than the full COPY/SELECT+INSERT rebuild, so skip the whole rebuild +# when the fingerprint matches the last run's AND a last-good $OUT_DB already exists. Fails OPEN: any error or +# missing piece while computing the fingerprint falls through to the existing full-rebuild path unchanged -- +# this is purely an optimization, never a new failure mode or a new way to serve stale data. +sqlite_source_fingerprint() { + [ -s "$APP_DB" ] || return 1 + fp="" + for spec in "pull_requests:updated_at" "review_audit:created_at" "review_targets:updated_at" "ai_usage_events:created_at"; do + tbl="${spec%%:*}" + col="${spec#*:}" + if source_table_exists "$tbl"; then + val="$(sqlite3 "$APP_DB" "SELECT COUNT(*) || ':' || COALESCE(MAX($col), '') FROM $tbl")" || return 1 + else + val="absent" + fi + fp="$fp;$tbl=$val" + done + printf '%s' "$fp" +} + +pg_source_fingerprint() { + fp="" + for spec in "pull_requests:updated_at" "review_audit:created_at" "review_targets:updated_at" "ai_usage_events:created_at"; do + tbl="${spec%%:*}" + col="${spec#*:}" + if pg_table_exists "$tbl"; then + val="$(pg_scalar "SELECT COUNT(*) || ':' || COALESCE(MAX($col)::text, '') FROM $tbl")" || return 1 + else + val="absent" + fi + fp="$fp;$tbl=$val" + done + printf '%s' "$fp" +} + +# Persist the fingerprint that was actually just exported, so the NEXT run's fast-path check above compares +# against real state. Written atomically (temp file + mv) and only when a fingerprint was actually computed -- +# never persists an empty/unknown value, which would otherwise let two unrelated "couldn't compute" runs +# falsely compare equal. +persist_fingerprint() { + [ -n "$CURRENT_FINGERPRINT" ] || return 0 + printf '%s' "$CURRENT_FINGERPRINT" >"${FINGERPRINT_FILE}.tmp" + mv "${FINGERPRINT_FILE}.tmp" "$FINGERPRINT_FILE" +} + mkdir -p "$OUT_DIR" +CURRENT_FINGERPRINT="" +if pg_enabled; then + if command -v psql >/dev/null 2>&1; then + pg_export_connection_env + CURRENT_FINGERPRINT="$(pg_source_fingerprint)" || CURRENT_FINGERPRINT="" + fi +else + CURRENT_FINGERPRINT="$(sqlite_source_fingerprint)" || CURRENT_FINGERPRINT="" +fi + +if [ -n "$CURRENT_FINGERPRINT" ] && [ -s "$OUT_DB" ] && [ -s "$FINGERPRINT_FILE" ] && [ "$(cat "$FINGERPRINT_FILE")" = "$CURRENT_FINGERPRINT" ]; then + echo "reporting export skipped: source unchanged since last export" + exit 0 +fi + rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm" TMP_DB_SQL="$(sql_string "$TMP_DB")" @@ -328,6 +393,7 @@ FROM ai_usage_events sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok" mv "$TMP_DB" "$OUT_DB" rm -f "$TMP_DB-wal" "$TMP_DB-shm" + persist_fingerprint echo "reporting export complete: $OUT_DB" exit 0 @@ -526,5 +592,6 @@ fi sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok" mv "$TMP_DB" "$OUT_DB" rm -f "$TMP_DB-wal" "$TMP_DB-shm" +persist_fingerprint echo "reporting export complete: $OUT_DB" diff --git a/test/unit/selfhost-grafana-reporting.test.ts b/test/unit/selfhost-grafana-reporting.test.ts index d84cf711d4..c589cfb960 100644 --- a/test/unit/selfhost-grafana-reporting.test.ts +++ b/test/unit/selfhost-grafana-reporting.test.ts @@ -28,8 +28,8 @@ function sqlite(db: string, sql: string): string { return execFileSync("sqlite3", [db, sql], { encoding: "utf8" }).trim(); } -function runExporter(root: string, sourceDb: string, outDb: string, env: Record = {}): void { - execFileSync("sh", ["scripts/export-grafana-reporting-db.sh"], { +function runExporter(root: string, sourceDb: string, outDb: string, env: Record = {}): string { + return execFileSync("sh", ["scripts/export-grafana-reporting-db.sh"], { cwd: process.cwd(), env: { ...process.env, @@ -39,6 +39,7 @@ function runExporter(root: string, sourceDb: string, outDb: string, env: Record< ...env, }, stdio: "pipe", + encoding: "utf8", }); } @@ -636,4 +637,78 @@ esac expect(existsSync(outDb)).toBe(false); expect(readdirSync(csvTmp)).toEqual([]); }); + + // ── Incremental fast-path (#3895) ────────────────────────────────────────────────────────────────── + it("skips the rebuild on a second run when the SQLite source is unchanged", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE pull_requests ( + repo_full_name TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, state TEXT NOT NULL, + author_login TEXT, merged_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at) + VALUES ('JSONbored/gittensory', 5001, 'unchanged PR', 'open', 'JSONbored', NULL, '2026-07-06T00:00:00Z', '2026-07-06T00:00:00Z'); + + CREATE TABLE review_audit ( + id TEXT NOT NULL, target_id TEXT NOT NULL, event_type TEXT NOT NULL, decision TEXT, + source TEXT NOT NULL, created_at TEXT NOT NULL + ); + `); + + const first = runExporter(root, appDb, outDb); + expect(first).toContain("reporting export complete"); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1"); + + const second = runExporter(root, appDb, outDb); + expect(second).toContain("reporting export skipped: source unchanged since last export"); + // The last-good snapshot is untouched, not silently emptied or corrupted by the skip. + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1"); + }); + + it("redoes the rebuild once the SQLite source actually changes, reflecting the new row", () => { + const root = tmpRoot(); + const appDb = join(root, "app.sqlite"); + const outDb = join(root, "reporting.sqlite"); + sqlite(appDb, ` + CREATE TABLE pull_requests ( + repo_full_name TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, state TEXT NOT NULL, + author_login TEXT, merged_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL + ); + INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at) + VALUES ('JSONbored/gittensory', 5002, 'first PR', 'open', 'JSONbored', NULL, '2026-07-06T00:00:00Z', '2026-07-06T00:00:00Z'); + + CREATE TABLE review_audit ( + id TEXT NOT NULL, target_id TEXT NOT NULL, event_type TEXT NOT NULL, decision TEXT, + source TEXT NOT NULL, created_at TEXT NOT NULL + ); + `); + runExporter(root, appDb, outDb); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1"); + + sqlite(appDb, ` + INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at) + VALUES ('JSONbored/gittensory', 5003, 'second PR', 'open', 'JSONbored', NULL, '2026-07-06T00:05:00Z', '2026-07-06T00:05:00Z'); + `); + const second = runExporter(root, appDb, outDb); + expect(second).toContain("reporting export complete"); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("2"); + }); + + it("skips the rebuild on a second run when the Postgres source is unchanged", () => { + const root = tmpRoot(); + const outDb = join(root, "reporting.sqlite"); + const bin = fakePsql(root); + const runOpts = { DATABASE_URL: "postgres://gittensory:pw@postgres:5432/gittensory", PATH: `${bin}:${process.env.PATH ?? ""}` }; + + const first = runExporter(root, join(root, "unused.sqlite"), outDb, runOpts); + expect(first).toContain("reporting export complete"); + + const second = runExporter(root, join(root, "unused.sqlite"), outDb, runOpts); + expect(second).toContain("reporting export skipped: source unchanged since last export"); + expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok"); + expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("3"); + }); });