JSONbored · loopover-orb · Jul 7, 2026 · Jul 7, 2026
@@ -6,6 +6,7 @@ PG_DB="${GITTENSORY_REPORTING_SOURCE_DATABASE_URL:-${DATABASE_URL:-}}"
 OUT_DIR="${GITTENSORY_REPORTING_DIR:-/reporting}"
 OUT_DB="${GITTENSORY_REPORTING_DB:-$OUT_DIR/gittensory-reporting.sqlite}"
 TMP_DB="${OUT_DB}.tmp"
+FINGERPRINT_FILE="${OUT_DB}.fingerprint"
 CSV_TMP_DIR="$(mktemp -d)"
 
 cleanup() {
@@ -134,8 +135,72 @@ sqlite_import_csv() {
 SQL
 }
 
+# Incremental fast-path (#3895): a live review pipeline is bursty -- most 30s cycles change nothing since the
+# last export, yet the full rebuild below re-exports and re-imports every row of every table every time. That
+# cost grows without bound as ai_usage_events (an append-only log of every AI call) accumulates -- measured on
+# a real self-host instance at 91GB of cumulative block I/O in 37 hours. A cheap COUNT+MAX fingerprint per
+# table is orders of magnitude cheaper than the full COPY/SELECT+INSERT rebuild, so skip the whole rebuild
+# when the fingerprint matches the last run's AND a last-good $OUT_DB already exists. Fails OPEN: any error or
+# missing piece while computing the fingerprint falls through to the existing full-rebuild path unchanged --
+# this is purely an optimization, never a new failure mode or a new way to serve stale data.
+sqlite_source_fingerprint() {
+  [ -s "$APP_DB" ] || return 1
+  fp=""
+  for spec in "pull_requests:updated_at" "review_audit:created_at" "review_targets:updated_at" "ai_usage_events:created_at"; do
+    tbl="${spec%%:*}"
+    col="${spec#*:}"
+    if source_table_exists "$tbl"; then
+      val="$(sqlite3 "$APP_DB" "SELECT COUNT(*) || ':' || COALESCE(MAX($col), '') FROM $tbl")" || return 1
+    else
+      val="absent"
+    fi
+    fp="$fp;$tbl=$val"
+  done
+  printf '%s' "$fp"
+}
+
+pg_source_fingerprint() {
+  fp=""
+  for spec in "pull_requests:updated_at" "review_audit:created_at" "review_targets:updated_at" "ai_usage_events:created_at"; do
+    tbl="${spec%%:*}"
+    col="${spec#*:}"
+    if pg_table_exists "$tbl"; then
+      val="$(pg_scalar "SELECT COUNT(*) || ':' || COALESCE(MAX($col)::text, '') FROM $tbl")" || return 1
+    else
+      val="absent"
+    fi
+    fp="$fp;$tbl=$val"
+  done
+  printf '%s' "$fp"
+}
+
+# Persist the fingerprint that was actually just exported, so the NEXT run's fast-path check above compares
+# against real state. Written atomically (temp file + mv) and only when a fingerprint was actually computed --
+# never persists an empty/unknown value, which would otherwise let two unrelated "couldn't compute" runs
+# falsely compare equal.
+persist_fingerprint() {
+  [ -n "$CURRENT_FINGERPRINT" ] || return 0
+  printf '%s' "$CURRENT_FINGERPRINT" >"${FINGERPRINT_FILE}.tmp"
+  mv "${FINGERPRINT_FILE}.tmp" "$FINGERPRINT_FILE"
+}
+
 mkdir -p "$OUT_DIR"
 
+CURRENT_FINGERPRINT=""
+if pg_enabled; then
+  if command -v psql >/dev/null 2>&1; then
+    pg_export_connection_env
+    CURRENT_FINGERPRINT="$(pg_source_fingerprint)" || CURRENT_FINGERPRINT=""
+  fi
+else
+  CURRENT_FINGERPRINT="$(sqlite_source_fingerprint)" || CURRENT_FINGERPRINT=""
+fi
+
+if [ -n "$CURRENT_FINGERPRINT" ] && [ -s "$OUT_DB" ] && [ -s "$FINGERPRINT_FILE" ] && [ "$(cat "$FINGERPRINT_FILE")" = "$CURRENT_FINGERPRINT" ]; then
+  echo "reporting export skipped: source unchanged since last export"
+  exit 0
+fi
+
 rm -f "$TMP_DB" "$TMP_DB-wal" "$TMP_DB-shm"
 TMP_DB_SQL="$(sql_string "$TMP_DB")"
 
@@ -328,6 +393,7 @@ FROM ai_usage_events
   sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok"
   mv "$TMP_DB" "$OUT_DB"
   rm -f "$TMP_DB-wal" "$TMP_DB-shm"
+  persist_fingerprint
 
   echo "reporting export complete: $OUT_DB"
   exit 0
@@ -526,5 +592,6 @@ fi
 sqlite3 "$TMP_DB" "PRAGMA quick_check;" | grep -qx "ok"
 mv "$TMP_DB" "$OUT_DB"
 rm -f "$TMP_DB-wal" "$TMP_DB-shm"
+persist_fingerprint
 
 echo "reporting export complete: $OUT_DB"
@@ -28,8 +28,8 @@ function sqlite(db: string, sql: string): string {
   return execFileSync("sqlite3", [db, sql], { encoding: "utf8" }).trim();
 }
 
-function runExporter(root: string, sourceDb: string, outDb: string, env: Record<string, string> = {}): void {
-  execFileSync("sh", ["scripts/export-grafana-reporting-db.sh"], {
+function runExporter(root: string, sourceDb: string, outDb: string, env: Record<string, string> = {}): string {
+  return execFileSync("sh", ["scripts/export-grafana-reporting-db.sh"], {
     cwd: process.cwd(),
     env: {
       ...process.env,
@@ -39,6 +39,7 @@ function runExporter(root: string, sourceDb: string, outDb: string, env: Record<
       ...env,
     },
     stdio: "pipe",
+    encoding: "utf8",
   });
 }
 
@@ -636,4 +637,78 @@ esac
     expect(existsSync(outDb)).toBe(false);
     expect(readdirSync(csvTmp)).toEqual([]);
   });
+
+  // ── Incremental fast-path (#3895) ──────────────────────────────────────────────────────────────────
+  it("skips the rebuild on a second run when the SQLite source is unchanged", () => {
+    const root = tmpRoot();
+    const appDb = join(root, "app.sqlite");
+    const outDb = join(root, "reporting.sqlite");
+    sqlite(appDb, `
+      CREATE TABLE pull_requests (
+        repo_full_name TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, state TEXT NOT NULL,
+        author_login TEXT, merged_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
+      );
+      INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at)
+      VALUES ('JSONbored/gittensory', 5001, 'unchanged PR', 'open', 'JSONbored', NULL, '2026-07-06T00:00:00Z', '2026-07-06T00:00:00Z');
+
+      CREATE TABLE review_audit (
+        id TEXT NOT NULL, target_id TEXT NOT NULL, event_type TEXT NOT NULL, decision TEXT,
+        source TEXT NOT NULL, created_at TEXT NOT NULL
+      );
+    `);
+
+    const first = runExporter(root, appDb, outDb);
+    expect(first).toContain("reporting export complete");
+    expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1");
+
+    const second = runExporter(root, appDb, outDb);
+    expect(second).toContain("reporting export skipped: source unchanged since last export");
+    // The last-good snapshot is untouched, not silently emptied or corrupted by the skip.
+    expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok");
+    expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1");
+  });
+
+  it("redoes the rebuild once the SQLite source actually changes, reflecting the new row", () => {
+    const root = tmpRoot();
+    const appDb = join(root, "app.sqlite");
+    const outDb = join(root, "reporting.sqlite");
+    sqlite(appDb, `
+      CREATE TABLE pull_requests (
+        repo_full_name TEXT NOT NULL, number INTEGER NOT NULL, title TEXT NOT NULL, state TEXT NOT NULL,
+        author_login TEXT, merged_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
+      );
+      INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at)
+      VALUES ('JSONbored/gittensory', 5002, 'first PR', 'open', 'JSONbored', NULL, '2026-07-06T00:00:00Z', '2026-07-06T00:00:00Z');
+
+      CREATE TABLE review_audit (
+        id TEXT NOT NULL, target_id TEXT NOT NULL, event_type TEXT NOT NULL, decision TEXT,
+        source TEXT NOT NULL, created_at TEXT NOT NULL
+      );
+    `);
+    runExporter(root, appDb, outDb);
+    expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("1");
+
+    sqlite(appDb, `
+      INSERT INTO pull_requests (repo_full_name, number, title, state, author_login, merged_at, created_at, updated_at)
+      VALUES ('JSONbored/gittensory', 5003, 'second PR', 'open', 'JSONbored', NULL, '2026-07-06T00:05:00Z', '2026-07-06T00:05:00Z');
+    `);
+    const second = runExporter(root, appDb, outDb);
+    expect(second).toContain("reporting export complete");
+    expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("2");
+  });
+
+  it("skips the rebuild on a second run when the Postgres source is unchanged", () => {
+    const root = tmpRoot();
+    const outDb = join(root, "reporting.sqlite");
+    const bin = fakePsql(root);
+    const runOpts = { DATABASE_URL: "postgres://gittensory:pw@postgres:5432/gittensory", PATH: `${bin}:${process.env.PATH ?? ""}` };
+
+    const first = runExporter(root, join(root, "unused.sqlite"), outDb, runOpts);
+    expect(first).toContain("reporting export complete");
+
+    const second = runExporter(root, join(root, "unused.sqlite"), outDb, runOpts);
+    expect(second).toContain("reporting export skipped: source unchanged since last export");
+    expect(sqlite(outDb, "PRAGMA quick_check;")).toBe("ok");
+    expect(sqlite(outDb, "SELECT count(*) FROM review_targets;")).toBe("3");
+  });
 });