fix(observability): avoid exposing live AMS ledgers

.env.example

@@ -495,6 +495,14 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS=30  # refresh cadence for the redacted reporting SQLite export
 # GITTENSORY_REPORTING_SOURCE_DB=/appdb/gittensory.sqlite  # if DATABASE_PATH=/data/custom.sqlite, set /appdb/custom.sqlite
 #
+# AMS (gittensory-miner) ledger dashboards — only useful when a miner ALSO runs on this same host (see
+# packages/gittensory-miner/docs/observability.md). Requires --profile ams-observability, separate from the
+# general observability profile above: an engine-only deployment has nothing for this exporter to read.
+# Grafana never mounts the live ledgers directly — this exporter reads them read-only and writes a redacted
+# snapshot (free-form attempt_log_events.reason/.payload_json dropped) into the same reporting volume.
+# GITTENSORY_MINER_CONFIG_DIR=~/.config/gittensory-miner  # host dir the exporter mounts read-only at /ams-ledgers
+# GITTENSORY_AMS_REPORTING_EXPORT_INTERVAL_SECONDS=30    # refresh cadence for the redacted AMS ledger exports
+#
 # Claude usage telemetry → OTEL collector → Prometheus → the Claude usage dashboard. OFF by default.
 # CLAUDE_CODE_ENABLE_TELEMETRY=1             # enable; needs --profile observability (starts the otel-collector)
 # OTEL_TRACES_EXPORTER=otlp                  # also enables app job/provider spans → Tempo; empty = no app traces

docker-compose.yml

@@ -835,6 +835,56 @@ services:
       start_period: 30s
       retries: 3
 
+  # Redacted AMS (gittensory-miner) ledger export -- a DEDICATED profile, not bundled into `observability`: unlike
+  # the engine's own reporting-exporter above, this only does anything useful when a miner is ALSO running on this
+  # same host with its ledgers under GITTENSORY_MINER_CONFIG_DIR (an engine-only maintainer deployment has nothing
+  # for it to read). Grafana must never mount that directory directly -- attempt_log_events.reason/.payload_json
+  # are free-form and can carry arbitrary internal detail (#5184 follow-up / PR #5471's flagged confidentiality
+  # gap) -- so this exporter reads the live ledgers read-only and writes a redacted snapshot into the SAME shared
+  # `grafana-reporting-data:/reporting` volume the engine's own exporter already writes to; only Grafana's
+  # ams-ledgers.yml datasources (pointed at /reporting/ams-*.sqlite) ever read from there.
+  ams-reporting-exporter:
+    image: alpine:3.20
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["ams-observability"]
+    volumes:
+      # Read-only live-ledger access for the exporter only -- Grafana does not get this mount (mirrors the
+      # /appdb:ro pattern above). Point this at the SAME host directory your miner uses as GITTENSORY_MINER_CONFIG_DIR.
+      - "${GITTENSORY_MINER_CONFIG_DIR:-~/.config/gittensory-miner}:/ams-ledgers:ro"
+      - grafana-reporting-data:/reporting
+      # Directory bind mount, not a single-file one -- see reporting-exporter's own comment above for why
+      # (a single-file mount pins to the pre-`git pull` inode and silently never picks up a script update).
+      - ./scripts:/scripts:ro
+    environment:
+      GITTENSORY_AMS_ATTEMPT_LOG_SOURCE_DB: "${GITTENSORY_AMS_ATTEMPT_LOG_SOURCE_DB:-/ams-ledgers/attempt-log.sqlite3}"
+      GITTENSORY_AMS_PREDICTION_LEDGER_SOURCE_DB: "${GITTENSORY_AMS_PREDICTION_LEDGER_SOURCE_DB:-/ams-ledgers/prediction-ledger.sqlite3}"
+      GITTENSORY_REPORTING_DIR: /reporting
+      GITTENSORY_AMS_ATTEMPT_LOG_REPORTING_DB: "${GITTENSORY_AMS_ATTEMPT_LOG_REPORTING_DB:-/reporting/ams-attempt-log.sqlite}"
+      GITTENSORY_AMS_PREDICTION_LEDGER_REPORTING_DB: "${GITTENSORY_AMS_PREDICTION_LEDGER_REPORTING_DB:-/reporting/ams-prediction-ledger.sqlite}"
+      GITTENSORY_AMS_REPORTING_EXPORT_INTERVAL_SECONDS: "${GITTENSORY_AMS_REPORTING_EXPORT_INTERVAL_SECONDS:-30}"
+    command:
+      - /bin/sh
+      - -c
+      - >-
+        apk add --no-cache sqlite >/dev/null 2>&1 &&
+        while true; do
+          sh /scripts/export-ams-reporting-db.sh || echo '[ams-reporting] export failed';
+          interval="$${GITTENSORY_AMS_REPORTING_EXPORT_INTERVAL_SECONDS:-30}";
+          case "$$interval" in ''|*[!0-9]*) interval=30;; esac;
+          sleep "$$interval";
+        done
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "db=\"$${GITTENSORY_AMS_ATTEMPT_LOG_REPORTING_DB:-/reporting/ams-attempt-log.sqlite}\"; test -s \"$$db\" && sqlite3 \"$$db\" 'PRAGMA quick_check;' | grep -q '^ok$'",
+        ]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
+
   # ── Log pipeline (--profile observability) ─────────────────────────────────
   # Loki stores logs; Promtail discovers every container in this compose project via the read-only
   # docker-proxy (NOT a raw socket) and ships their logs to Loki. Browse in Grafana → Explore →

grafana/provisioning/datasources/ams-ledgers.yml

@@ -1,16 +1,13 @@
-# AMS (gittensory-miner) local SQLite ledger datasources (#5184).
+# AMS (gittensory-miner) redacted SQLite reporting datasources (#5184).
 #
-# The miner keeps append-only SQLite ledgers under GITTENSORY_MINER_CONFIG_DIR (default ~/.config/gittensory-miner
-# on a laptop, or /data/miner in the fleet Docker image). This wires two scrape-free `frser-sqlite-datasource`
-# entries so a follow-on AMS usage dashboard can query them read-only — mirroring the maintainer `GittensoryDB`
-# entry in sqlite.yml. This file is purely additive (it touches no existing datasource) and provisions NO
-# dashboards or queries — that is the follow-on dashboard issue's scope.
+# Grafana must not mount the miner's live GITTENSORY_MINER_CONFIG_DIR ledgers. The attempt log and prediction
+# ledger contain free-form reasons, payload JSON, repo/target identifiers, readiness scores, and blocker/warning
+# codes that are private AMS intelligence. Instead, expose only redacted reporting-export snapshots, mirroring the
+# maintainer GittensoryDB pattern in sqlite.yml.
 #
-# PATH — the absolute path varies per install, so `path:` below is a documented placeholder. Mount your miner's
-# ledger directory (GITTENSORY_MINER_CONFIG_DIR) into the Grafana container at /ams-ledgers, e.g.:
-#   volumes: [ "${GITTENSORY_MINER_CONFIG_DIR:-~/.config/gittensory-miner}:/ams-ledgers:ro" ]
-# then Grafana reads the files below. The `:ro` mount + this read-only query plugin mean Grafana can never write
-# to the live ledgers. Adjust the two `path:` values if you mount elsewhere.
+# PATH — the absolute path varies per install, so `path:` below is a documented placeholder. Publish sanitized AMS
+# reporting exports into Grafana's existing reporting mount (for example /reporting/ams-attempt-log.sqlite and
+# /reporting/ams-prediction-ledger.sqlite) and keep the live ledger directory private to the miner.
 apiVersion: 1
 datasources:
   - name: AMS Attempt Log
@@ -19,11 +16,11 @@ datasources:
     access: proxy
     editable: false
     jsonData:
-      path: /ams-ledgers/attempt-log.sqlite3
+      path: /reporting/ams-attempt-log.sqlite
   - name: AMS Prediction Ledger
     type: frser-sqlite-datasource
     uid: ams-prediction-ledger
     access: proxy
     editable: false
     jsonData:
-      path: /ams-ledgers/prediction-ledger.sqlite3
+      path: /reporting/ams-prediction-ledger.sqlite

packages/gittensory-miner/docs/observability.md

@@ -1,7 +1,8 @@
 # Observing your miner
 
-How to point Grafana at a running miner's local SQLite ledgers to see its attempt and prediction history. This
-covers the **miner-specific** observability wiring only; for general self-host operations, see your ops runbook.
+How to point Grafana at redacted miner reporting exports to see attempt and prediction history without exposing the
+miner's live local ledgers. This covers the **miner-specific** observability wiring only; for general self-host
+operations, see your ops runbook.
 
 ## What's observable
 
@@ -13,40 +14,47 @@ The miner writes append-only SQLite ledgers under `GITTENSORY_MINER_CONFIG_DIR`
   timestamps), table `attempt_log_events`.
 - **`prediction-ledger.sqlite3`** — recorded predicted-gate verdicts for later scoring.
 
-## Point Grafana at the ledgers
+Those live files can contain free-form payloads, repo/target identifiers, readiness scores, and blocker/warning
+codes. Keep `GITTENSORY_MINER_CONFIG_DIR` private to the miner. Grafana should read only sanitized reporting
+exports that operators create from those ledgers.
+
+## Point Grafana at reporting exports
 
 The repo ships datasource provisioning at
 [`grafana/provisioning/datasources/ams-ledgers.yml`](../../../grafana/provisioning/datasources/ams-ledgers.yml)
 — two **read-only** `frser-sqlite-datasource` entries: `AMS Attempt Log` (uid `ams-attempt-log`) and
-`AMS Prediction Ledger` (uid `ams-prediction-ledger`).
+`AMS Prediction Ledger` (uid `ams-prediction-ledger`). Their default paths live under Grafana's reporting mount,
+not under the miner config directory.
 
 1. **Install the SQLite plugin** in Grafana — the same one the maintainer `GittensoryDB` datasource uses:
 
    ```sh
    GF_INSTALL_PLUGINS=frser-sqlite-datasource
    ```
 
-2. **Mount your ledger directory** into the Grafana container, read-only, at `/ams-ledgers` so the provisioned
-   `path:` values resolve (the `:ro` mount plus the query-only plugin mean Grafana can never write the live
-   ledgers):
+2. **Run the AMS reporting exporter**, a dedicated compose profile (only useful when a miner also runs on this
+   same host — an engine-only deployment has nothing for it to read):
 
-   ```yaml
-   # in your Grafana service (docker-compose)
-   volumes:
-     - "${GITTENSORY_MINER_CONFIG_DIR:-~/.config/gittensory-miner}:/ams-ledgers:ro"
+   ```sh
+   docker compose --profile ams-observability up -d
    ```
 
-   The two datasources point at `/ams-ledgers/attempt-log.sqlite3` and `/ams-ledgers/prediction-ledger.sqlite3`.
-   If you mount elsewhere, edit the two `path:` values in `ams-ledgers.yml` to match.
+   Set `GITTENSORY_MINER_CONFIG_DIR` in your `.env` (see [`.env.example`](../../../.env.example)) to the same
+   directory your miner uses. The `ams-reporting-exporter` container mounts it **read-only**, runs
+   [`scripts/export-ams-reporting-db.sh`](../../../scripts/export-ams-reporting-db.sh) on an interval
+   (`GITTENSORY_AMS_REPORTING_EXPORT_INTERVAL_SECONDS`, default 30s), and writes the redacted snapshots into the
+   same `reporting` volume Grafana already reads — Grafana itself never mounts the live ledgers. The exported
+   schema drops `attempt_log_events.reason`/`.payload_json` (the free-form fields) entirely; every other column,
+   including the `predictions` table's `blocker_codes_json`/`warning_codes_json` (fixed, engine-defined codes —
+   never free text), passes through unchanged.
 
 3. **Restart Grafana.** The two datasources appear under **Connections → Data sources**, already provisioned
    (non-editable) so they survive restarts.
 
 ## Load a dashboard
 
 Dashboards live in [`grafana/dashboards/`](../../../grafana/dashboards/) and are auto-provisioned from that
-directory. To visualize the ledgers, add an AMS dashboard JSON there — or import one at runtime via the Grafana
-UI (**Dashboards → Import**) — and point its panels at the `AMS Attempt Log` / `AMS Prediction Ledger`
-datasources above. Panels query the ledger tables directly (e.g. `SELECT * FROM attempt_log_events`); the
-`frser-sqlite-datasource` plugin also supports `json_extract(payload_json, '$.…')` to read fields nested inside
-an event's payload.
+directory. To visualize AMS activity, add a dashboard JSON there — or import one at runtime via the Grafana UI
+(**Dashboards → Import**) — and point its panels at the `AMS Attempt Log` / `AMS Prediction Ledger` datasources
+above. Panels should query only the redacted reporting schema (e.g. `SELECT * FROM attempt_log_events`), never a
+`payload_json`/`reason` column — the exporter drops both, so a panel referencing them returns no such column.

scripts/export-ams-reporting-db.sh

@@ -0,0 +1,185 @@
+#!/bin/sh
+set -eu
+
+# AMS (gittensory-miner) redacted reporting export (#5184 follow-up; closes the gap PR #5471 flagged: Grafana must
+# never mount the miner's live GITTENSORY_MINER_CONFIG_DIR ledgers directly -- attempt_log_events.reason and
+# .payload_json are free-form and can carry arbitrary internal detail). Mirrors export-grafana-reporting-db.sh's
+# shape (incremental fingerprint fast-path, atomic tmp-then-move, fail-open on a missing/unreadable source) but is
+# deliberately a SEPARATE script: the two AMS ledgers are SQLite-only (the miner has no Postgres mode) and each
+# source table is INSERT-only (attempt_log_events' own header states this invariant; prediction-ledger.js's is the
+# same), so neither needs the mutable-table full-content-hash path the main script carries for pull_requests/
+# review_targets.
+#
+# Exports TWO independent ledgers in one run -- a missing/corrupt attempt log must never block the prediction
+# ledger's export or vice versa, so each runs its own fail-open pass.
+#
+# Bump whenever this script's own mapping/redaction logic changes (not just when a source table gains a column):
+# the incremental fast-path below only fingerprints SOURCE ROW COUNT + latest timestamp, so a logic-only edit
+# would otherwise serve the previous run's output forever.
+SCRIPT_VERSION="${GITTENSORY_AMS_REPORTING_SCRIPT_VERSION:-1}"
+
+OUT_DIR="${GITTENSORY_REPORTING_DIR:-/reporting}"
+ATTEMPT_LOG_SOURCE_DB="${GITTENSORY_AMS_ATTEMPT_LOG_SOURCE_DB:-/ams-ledgers/attempt-log.sqlite3}"
+ATTEMPT_LOG_OUT_DB="${GITTENSORY_AMS_ATTEMPT_LOG_REPORTING_DB:-$OUT_DIR/ams-attempt-log.sqlite}"
+PREDICTION_LEDGER_SOURCE_DB="${GITTENSORY_AMS_PREDICTION_LEDGER_SOURCE_DB:-/ams-ledgers/prediction-ledger.sqlite3}"
+PREDICTION_LEDGER_OUT_DB="${GITTENSORY_AMS_PREDICTION_LEDGER_REPORTING_DB:-$OUT_DIR/ams-prediction-ledger.sqlite}"
+
+mkdir -p "$OUT_DIR"
+
+hash_stdin() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum | awk '{print $1}'
+  elif command -v shasum >/dev/null 2>&1; then
+    shasum -a 256 | awk '{print $1}'
+  else
+    cat >/dev/null
+    return 1
+  fi
+}
+
+source_table_exists() {
+  db="$1"
+  tbl="$2"
+  sqlite3 "$db" "SELECT 1 FROM sqlite_master WHERE type='table' AND name='$tbl' LIMIT 1" | grep -q 1
+}
+
+# Insert-only source: a row COUNT + MAX(created_at) aggregate can never miss a real change (nothing UPDATEs a row
+# in place, matching attempt_log_events'/predictions' own append-only invariants), and stays O(1)-ish instead of
+# an O(row-count) full-table hash as each ledger grows without bound.
+append_only_fingerprint() {
+  db="$1"
+  tbl="$2"
+  time_col="$3"
+  sqlite3 "$db" "SELECT COUNT(*) || ':' || COALESCE(MAX($time_col), '') FROM $tbl"
+}
+
+reporting_db_ok() {
+  db="$1"
+  [ -s "$db" ] || return 1
+  sqlite3 "$db" "PRAGMA quick_check;" 2>/dev/null | grep -qx "ok"
+}
+
+persist_fingerprint() {
+  fingerprint="$1"
+  file="$2"
+  [ -n "$fingerprint" ] || return 0
+  printf '%s' "$fingerprint" >"${file}.tmp"
+  mv "${file}.tmp" "$file"
+}
+
+# One ledger's full fail-open/fingerprint/atomic-export pass. Never propagates a failure to the caller (this
+# script exports two independent ledgers per run and a bad one must not block the other) -- always returns 0,
+# logging to stderr on any skip/failure path.
+#
+# $1 label (for log lines)   $2 source db   $3 out db   $4 source table   $5 time column
+# $6 redacted CREATE TABLE DDL   $7 redacted SELECT column list (source-table column names, in DDL column order)
+export_ledger() {
+  label="$1"
+  src="$2"
+  out="$3"
+  tbl="$4"
+  time_col="$5"
+  ddl="$6"
+  select_cols="$7"
+
+  tmp="${out}.tmp"
+  fp_file="${out}.fingerprint"
+  rm -f "$tmp" "$tmp-wal" "$tmp-shm"
+
+  if [ ! -s "$src" ]; then
+    if [ -s "$out" ]; then
+      echo "[ams-reporting:$label] export skipped: source missing at $src; preserving last-good $out" >&2
+    else
+      sqlite3 "$tmp" "$ddl"
+      sqlite3 "$tmp" "PRAGMA quick_check;" | grep -qx "ok"
+      mv "$tmp" "$out"
+      rm -f "$tmp-wal" "$tmp-shm"
+      echo "[ams-reporting:$label] export empty: source missing at $src" >&2
+    fi
+    return 0
+  fi
+
+  if ! source_table_exists "$src" "$tbl"; then
+    if [ -s "$out" ]; then
+      echo "[ams-reporting:$label] export skipped: table $tbl absent in $src; preserving last-good $out" >&2
+    else
+      sqlite3 "$tmp" "$ddl"
+      sqlite3 "$tmp" "PRAGMA quick_check;" | grep -qx "ok"
+      mv "$tmp" "$out"
+      rm -f "$tmp-wal" "$tmp-shm"
+      echo "[ams-reporting:$label] export empty: table $tbl absent in $src" >&2
+    fi
+    return 0
+  fi
+
+  fingerprint="script=$SCRIPT_VERSION;$(append_only_fingerprint "$src" "$tbl" "$time_col")"
+  if reporting_db_ok "$out" && [ -s "$fp_file" ] && [ "$(cat "$fp_file")" = "$fingerprint" ]; then
+    echo "[ams-reporting:$label] export skipped: source unchanged since last export"
+    return 0
+  fi
+
+  sqlite3 "$tmp" "$ddl"
+  out_sql="$(printf "%s" "$tmp" | sed "s/'/''/g")"
+  sqlite3 -cmd ".timeout 5000" "$src" "
+ATTACH '$out_sql' AS report;
+INSERT INTO report.$tbl SELECT $select_cols FROM main.$tbl;
+DETACH report;
+"
+  if ! sqlite3 "$tmp" "PRAGMA quick_check;" | grep -qx "ok"; then
+    rm -f "$tmp" "$tmp-wal" "$tmp-shm"
+    echo "[ams-reporting:$label] export failed: rebuilt database failed quick_check, preserving last-good $out" >&2
+    return 0
+  fi
+  mv "$tmp" "$out"
+  rm -f "$tmp-wal" "$tmp-shm"
+  persist_fingerprint "$fingerprint" "$fp_file"
+  echo "[ams-reporting:$label] export complete: $out"
+}
+
+# attempt_log_events: DROP `reason` and `payload_json` -- both free-form (payload_json in particular can nest
+# arbitrary per-event-type detail, up to and including file paths/diffs/prompt fragments). Every other column is
+# a bounded-vocabulary identifier/enum/timestamp, safe for a shared reporting export.
+export_ledger \
+  "attempt-log" \
+  "$ATTEMPT_LOG_SOURCE_DB" \
+  "$ATTEMPT_LOG_OUT_DB" \
+  "attempt_log_events" \
+  "created_at" \
+  "CREATE TABLE attempt_log_events (
+    id INTEGER PRIMARY KEY,
+    seq INTEGER NOT NULL,
+    attempt_id TEXT NOT NULL,
+    event_type TEXT NOT NULL,
+    action_class TEXT NOT NULL,
+    mode TEXT NOT NULL,
+    created_at TEXT NOT NULL
+  );
+  CREATE INDEX attempt_log_events_attempt_idx ON attempt_log_events(attempt_id, seq);
+  CREATE INDEX attempt_log_events_created_idx ON attempt_log_events(created_at);" \
+  "id, seq, attempt_id, event_type, action_class, mode, created_at"
+
+# predictions: kept as-is. Unlike attempt_log_events, every column here is already a bounded identifier, enum,
+# score, or a fixed-vocabulary code array (blocker_codes_json/warning_codes_json -- engine-defined codes, never
+# free text) -- exactly the kind of structured signal a "prediction ledger" dashboard needs to be useful at all.
+export_ledger \
+  "prediction-ledger" \
+  "$PREDICTION_LEDGER_SOURCE_DB" \
+  "$PREDICTION_LEDGER_OUT_DB" \
+  "predictions" \
+  "ts" \
+  "CREATE TABLE predictions (
+    id INTEGER PRIMARY KEY,
+    ts TEXT NOT NULL,
+    repo_full_name TEXT NOT NULL,
+    target_id INTEGER NOT NULL,
+    head_sha TEXT,
+    conclusion TEXT NOT NULL,
+    pack TEXT NOT NULL,
+    readiness_score REAL,
+    blocker_codes_json TEXT NOT NULL,
+    warning_codes_json TEXT NOT NULL,
+    engine_version TEXT NOT NULL
+  );
+  CREATE INDEX predictions_repo_idx ON predictions(repo_full_name, id);
+  CREATE INDEX predictions_ts_idx ON predictions(ts);" \
+  "id, ts, repo_full_name, target_id, head_sha, conclusion, pack, readiness_score, blocker_codes_json, warning_codes_json, engine_version"