From 435a5b0fd4bf6f0a69533fec2d85762b66649e04 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 13:13:12 -0700 Subject: [PATCH 01/19] fix(usage): catch the "" placeholder even without isApiErrorMessage Some builds emit Claude Code's dropped-connection placeholder turn (model: "", zero usage) without setting isApiErrorMessage, so it slipped past the exception filter and surfaced as a real \$0 "model in play" on the scorecard. The literal model marker is now checked alongside the flag, SCHEMA_VERSION bumps to 9 so cached sessions re-derive, and a regression test covers the flagless shape. Also re-anchors the usage-doc file:line citations shifted by this file's line movement (doc-citations gate). --- docs/TRANSCRIPTS.md | 6 +++--- docs/USAGE-SCORECARD-METRICS.md | 22 +++++++++---------- src/lib/usage-index.mjs | 17 +++++++++++---- tests/kit/usage-index.test.mjs | 38 +++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/docs/TRANSCRIPTS.md b/docs/TRANSCRIPTS.md index ce27c9c..1a8cb69 100644 --- a/docs/TRANSCRIPTS.md +++ b/docs/TRANSCRIPTS.md @@ -36,7 +36,7 @@ rewritten; rule 3 of the module header, `usage-index.mjs:22-29`): | Host | Store | Discovered by | |---|---|---| -| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:684`) — exactly one level of project directories | +| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:733`) — exactly one level of project directories | | Codex CLI | `~/.codex/sessions///
/rollout--.jsonl` | `listCodex` (`usage-index.mjs:705`) — the `yyyy/mm/dd` tree walk | Roots come from `defaultRoots()` (`usage-index.mjs:697-701`) and are injectable @@ -141,7 +141,7 @@ story is [Appendix A](#appendix-a--fix-history).) ### 3.2 `kind` — the attribution field -`userTurnKind` (`usage-index.mjs:451-455`) classifies every user-role turn: +`userTurnKind` (`usage-index.mjs:470-475`) classifies every user-role turn: | `kind` | Test | Meaning | |---|---|---| @@ -164,7 +164,7 @@ Two deliberate subtleties: - **`tool-result` outranks `context`**: a `tool_result` block on an `isMeta` entry is still tool feedback. -Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:584-592`) +Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:657`) — rollouts only record real prompts as `user_message` events (§1.2). Coverage: `tests/kit/usage-index.test.mjs` — "user-role turns carry a kind" diff --git a/docs/USAGE-SCORECARD-METRICS.md b/docs/USAGE-SCORECARD-METRICS.md index fb24edf..8ee9b9d 100644 --- a/docs/USAGE-SCORECARD-METRICS.md +++ b/docs/USAGE-SCORECARD-METRICS.md @@ -128,9 +128,9 @@ responses = Σ over included sessions of session.responses **Source:** - Filter: a parsed record with zero assistant turns is dropped entirely — "no - assistant turn → not a session" (`usage-index.mjs:923`) — and a record whose + assistant turn → not a session" (`usage-index.mjs:935`) — and a record whose last activity falls outside the requested window is dropped too - (`usage-index.mjs:924`). + (`usage-index.mjs:936`). - `responses` accumulation: Claude increments per assistant message (`usage-index.mjs:504-509`); Codex increments per `agent_message` event (`usage-index.mjs:651-655`). @@ -213,7 +213,7 @@ already in effect on the given day, comparing ISO date strings lexicographically so no `Date` parsing is involved and the module stays clock-free. -`aggregate()` passes each usage row's own `day` (`usage-index.mjs:911`), which +`aggregate()` passes each usage row's own `day` (`usage-index.mjs:926`), which it already has because rows are keyed by `(day, model)`. **This is the whole point:** tokens metered in August must still read as August's rate when the panel is opened in December. Pricing by *today's* date instead would restate a @@ -297,7 +297,7 @@ tokens = input + output + cacheRead + cacheWrite (summed across all rows in wi **Source:** `t.tokens` from `totals`, accumulated per row at `usage-index.mjs:935` (`rowTokens = row.input + row.output + row.cacheRead + row.cacheWrite`) and rolled into `totals.tokens` via `addTo` -(`usage-index.mjs:843-852`). Rendered with `fmtTok()` +(`usage-index.mjs:1027-1033`). Rendered with `fmtTok()` (`dashboard/client.mjs`): `≥1e9` → `"X.XB"`, `≥1e6` → `"X.XM"`, `≥1e3` → `"X.XK"`, else the rounded integer. @@ -309,7 +309,7 @@ numbers as percentages of `t.tokens` (`dashboard/client.mjs`, **What "input" excludes.** For both providers, the `input` counter recorded per row is **gross input minus cached input** — Claude's parser reads `cache_read_input_tokens` and `cache_creation_input_tokens` as separate fields -the provider already reports separately (`usage-index.mjs:523-524`); Codex's +the provider already reports separately (`usage-index.mjs:545-546`); Codex's parser subtracts `cached_input_tokens` from `input_tokens` explicitly (`usage-index.mjs:666-675`, `input: Math.max(0, gross - cacheRead)`) because Codex's own `input_tokens` field **includes** cached tokens and would @@ -396,7 +396,7 @@ session data, and each needs its own fix: human, or genuinely idle) donates its *entire* idle stretch to the span, even though no work happened during it. Fix: split each session into active sub-intervals wherever the gap between two consecutive timestamps - exceeds `IDLE_GAP_MS` (15 minutes, `usage-index.mjs:62`), then union + exceeds `IDLE_GAP_MS` (15 minutes, `usage-index.mjs:80`), then union *those* sub-intervals — this is `engagedSeconds`. **Source:** @@ -462,7 +462,7 @@ byDay[day].cost = Σ costOf(row) for every usage row whose day == that key **Source:** the day key is the row's own `row.day`, computed once at parse time as **local calendar day**, not UTC -(`usage-index.mjs:530`/`usage-index.mjs:667` call `localDay(at)`) — so a +(`usage-index.mjs:542`/`usage-index.mjs:679` call `localDay(at)`) — so a session that runs from 23:58 local to 00:05 local is billed to the day its *first* row landed on (test: `tests/kit/usage-index.test.mjs:634`, "a session that opens before midnight @@ -489,10 +489,10 @@ renders "no sessions in window" instead of zeroed figures **Formula:** identical aggregation to every other bucket (`byProvider[s.provider]`, populated via `addTo()`, `usage-index.mjs:889-898`, - called once per session at `usage-index.mjs:1000`), keyed by the literal string + called once per session at `usage-index.mjs:1031`), keyed by the literal string `"claude"` or `"codex"` assigned at parse time (`blankSession(id, 'claude')` / `blankSession(id, 'codex')`, -`usage-index.mjs:291-301`, `parseClaude`/`parseCodex` entry points). +`usage-index.mjs:345-355`, `parseClaude`/`parseCodex` entry points). **Why this pairing is the one under the most scrutiny.** Both providers' tokens are summed into the *same* `tokens`/`cost` fields using the *same* @@ -575,7 +575,7 @@ excluded subagent-replay session still shows up as "used," at zero cost, rather than vanishing. `byModel[...].responses` is populated from `row.responses` -(`usage-index.mjs:921`), which in turn comes from the `responses` field +(`usage-index.mjs:964`), which in turn comes from the `responses` field passed into `addUsage()` at the call site — `1` per Claude assistant turn (`usage-index.mjs:484-490`), or `rec.responses` (the session's whole response count) once per Codex session, passed at the single point Codex calls @@ -917,7 +917,7 @@ rollout replays the parent's entire token history (ccusage/ccusage#950 measured up to 91× inflation) — while the session record stays visible. The rollout's own `session_meta.thread_source` sniff remains as the fallback when the ledger is absent or migrated beyond recognition. Codex sessions also carry -`reasoningOutput` (`usage-index.mjs:677-678`) — reasoning tokens are a **subset** +`reasoningOutput` (`usage-index.mjs:686-687`) — reasoning tokens are a **subset** of output tokens and are annotation only, never added to any sum. ## 14. Known limitations, restated as a single checklist diff --git a/src/lib/usage-index.mjs b/src/lib/usage-index.mjs index 9594f45..c89cf39 100644 --- a/src/lib/usage-index.mjs +++ b/src/lib/usage-index.mjs @@ -64,8 +64,14 @@ import { * v8: Codex `session_meta.model_provider` is retained as observed inference * provider evidence. v7 caches discarded that field, so every Codex * record must be re-derived rather than continuing to display an - * avoidable "provider not recorded". */ -export const SCHEMA_VERSION = 8; + * avoidable "provider not recorded". + * v9: the dropped-connection/API-error placeholder turn is now recognized by + * its literal `model: ""` marker as well as + * `isApiErrorMessage: true` — some builds emit the placeholder without + * that flag set. A v8-cached session parsed from such a transcript still + * carries `""` in `models` and a $0 usage row, so it must be + * re-derived or the placeholder keeps showing as a real "model in play". */ +export const SCHEMA_VERSION = 9; /** Silence longer than this ends a stretch of engagement. A session is split * into active sub-intervals at gaps ABOVE this bound (exactly this much is not @@ -514,8 +520,11 @@ function parseClaude(raw, { id, dirName, withTurns = false }) { // always zero. It IS real engaged time (counted above), but it is not a // model attempt: excluded from `models`/cost attribution so it can never // appear as a $0 "model in play," and counted instead as an EXCEPTION so - // it stays visible rather than silently vanishing. - if (e.isApiErrorMessage === true) { + // it stays visible rather than silently vanishing. isApiErrorMessage isn't + // reliably set on every build that emits this placeholder, so the literal + // model marker is checked directly too — it's the one part of the shape + // that's never varied in observed transcripts. + if (e.isApiErrorMessage === true || e.message.model === '') { rec.exceptions++; if (withTurns) { turns.push({ diff --git a/tests/kit/usage-index.test.mjs b/tests/kit/usage-index.test.mjs index d51ba1f..ab47823 100644 --- a/tests/kit/usage-index.test.mjs +++ b/tests/kit/usage-index.test.mjs @@ -714,6 +714,44 @@ test('a dropped-connection turn (isApiErrorMessage) counts as an exception, neve assert.equal(agg.byModel['claude-opus-5'].tokens, 150); }); +test('a "" placeholder without isApiErrorMessage still counts as an exception, never a $0 model', async () => { + // Some builds emit the same dropped-connection placeholder without setting + // isApiErrorMessage — the literal model marker is the one part of the shape + // that never varies, so it must be caught on its own too. + _resetForTest(); + const sb = soloSandbox(); + const base = Date.parse('2026-08-06T10:00:00.000Z'); + const iso = (offMs) => new Date(base + offMs).toISOString(); + const lines = [ + { type: 'user', sessionId: 'jjjj0000', cwd: '/Users/me/proj', timestamp: iso(0), + message: { role: 'user', content: [{ type: 'text', text: 'turn 1' }] } }, + { type: 'assistant', sessionId: 'jjjj0000', cwd: '/Users/me/proj', timestamp: iso(1000), + message: { role: 'assistant', model: 'claude-sonnet-5', usage: { input_tokens: 100, output_tokens: 50 }, content: [] } }, + { type: 'user', sessionId: 'jjjj0000', cwd: '/Users/me/proj', timestamp: iso(2000), + message: { role: 'user', content: [{ type: 'text', text: 'turn 2' }] } }, + // Same placeholder shape as above, but isApiErrorMessage is absent. + { type: 'assistant', sessionId: 'jjjj0000', cwd: '/Users/me/proj', timestamp: iso(3000), + message: { + role: 'assistant', model: '', stop_reason: 'stop_sequence', + usage: { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + content: [{ type: 'text', text: 'API Error: Connection closed mid-response.' }], + } }, + ]; + fs.writeFileSync(path.join(sb.claude, 'jjjj0000.jsonl'), `${lines.map((l) => JSON.stringify(l)).join('\n')}\n`); + + const agg = await buildIndex(opts(sb)); + const s = byId(agg, 'jjjj0000'); + + assert.ok(s, 'session indexed'); + assert.equal(s.responses, 2, 'the error placeholder still counts as a real turn'); + assert.equal(s.exceptions, 1); + assert.deepEqual(s.models, ['claude-sonnet-5'], '"" never enters the models list'); + assert.equal(s.tokens, 150, 'only the real turn contributes tokens'); + + assert.equal(agg.totals.exceptions, 1); + assert.equal(agg.byModel[''], undefined, 'no $0 "" row is ever created'); +}); + test('punchcard buckets responses by dow-hour with Monday as 0', async () => { _resetForTest(); const sb = sandbox(); From 073f34261cbc3152017667b225f1c8ba1eaaff8f Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 13:13:23 -0700 Subject: [PATCH 02/19] feat(dashboard): date-windowed Observability History browsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observability's History scope was limited to the live tailer's moving window (256 newest transcript files, 100-session projection). It now browses retained sessions over an explicit calendar window — 1d, 7d, 14d (default), 1mo, 3mo, 6mo, 1y, all — like Usage's day chips. - discoverJsonl() gains an optional sinceMs mtime cutoff - LiveSessionsService.historySnapshot({sinceMs}): a one-shot scan with its own projection (never touches live tailer state), swept with all-zero windows so unterminated sessions read as stale, not live - GET /api/live/history?window=, same publicLivePayload scrubbing as /api/live; 501 when the service lacks historySnapshot - window chips in the History sub-nav; History renders from a separate state.historySnapshot bucket so live SSE deltas can never clobber it The UI harness also allowlists /api/live/intelligence EventSource teardown aborts (pre-existing flake — Chromium reports deliberate stream closes as ERR_ABORTED, same as the existing events/transcripts entries) and gives LIVE_STUB a historySnapshot. --- src/lib/dashboard-server.mjs | 33 ++++++++ src/lib/dashboard/live/client.mjs | 31 +++++--- src/lib/dashboard/page.mjs | 10 +++ src/lib/live/live-sessions-service.mjs | 84 +++++++++++++++++--- src/lib/live/native-transcript-discovery.mjs | 11 ++- tests/dashboard.test.cjs | 41 +++++++++- tests/kit/live-service.test.mjs | 43 ++++++++++ tests/ui/dashboard-ui.mjs | 13 ++- 8 files changed, 239 insertions(+), 27 deletions(-) diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index d4f20fe..58565e5 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -412,6 +412,20 @@ function clampInt(raw, fallback, min, max) { return Math.min(max, Math.max(min, n)); } +// Observability History's window control — day-count approximations (a +// calendar "month" is treated as 30 days, matching clampDays' own plain-day +// semantics rather than adding calendar-aware month math for a browse filter). +const HISTORY_WINDOW_DAYS = { + '1d': 1, '7d': 7, '14d': 14, '1mo': 30, '3mo': 90, '6mo': 180, '1y': 365, +}; +/** ?window= → an epoch-ms cutoff, or null for "all time". Unknown/ + * missing tokens fall back to 14d, the same default clampDays uses. */ +function windowToSinceMs(raw, now = Date.now()) { + if (raw === 'all') return null; + const days = Object.hasOwn(HISTORY_WINDOW_DAYS, raw) ? HISTORY_WINDOW_DAYS[raw] : HISTORY_WINDOW_DAYS['14d']; + return now - days * 86_400_000; +} + function sendJson(res, status, payload) { res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store', 'x-content-type-options': 'nosniff' }); res.end(JSON.stringify(payload)); @@ -765,6 +779,25 @@ export function startDashboard({ return; } + if (url === '/api/live/history') { + // On-demand, not part of the SSE stream: a fresh scan per request, same + // privacy scrubbing (publicLivePayload) as every other live surface. + try { + const service = await getLive(); + if (typeof service.historySnapshot !== 'function') { + sendJson(res, 501, { error: 'history browsing not supported by this live service' }); + return; + } + const sinceMs = windowToSinceMs(query.get('window')); + sendJson(res, 200, publicLivePayload(await service.historySnapshot({ sinceMs }))); + } catch { + sendJson(res, 503, { error: 'live telemetry unavailable' }); + } finally { + scheduleLiveIdle(); + } + return; + } + if (url === '/api/live/events') { // Reserve the client-cap slot BEFORE the first await (getLive() may do a // dynamic import() + service.start()). Concurrent requests arriving diff --git a/src/lib/dashboard/live/client.mjs b/src/lib/dashboard/live/client.mjs index f76e68a..0aed186 100644 --- a/src/lib/dashboard/live/client.mjs +++ b/src/lib/dashboard/live/client.mjs @@ -20,7 +20,7 @@ export const LIVE_JS = ` function dashSseUrl(u){return DASH_TOKEN?u+(u.indexOf("?")<0?"?":"&")+"token="+encodeURIComponent(DASH_TOKEN):u;} var PAUSE_LIMIT=256,MAX_TURNS=500,TRANSCRIPT_COLLAPSE_KEY="ak-dash-transcript-collapsed",TERMINAL={completed:1,failed:1,cancelled:1}; function storedTranscriptCollapsed(){try{return localStorage.getItem(TRANSCRIPT_COLLAPSE_KEY)==="true";}catch(_){return false;}} - var state={snapshot:{schemaVersion:2,cursor:null,projects:[],sessions:[]},events:[],project:null,selected:null,node:null,browserLevel:"projects",scope:"live",paused:false,pending:[],overflow:false,source:null,resyncing:false,lastAt:0,active:false,connection:{key:"connecting",text:"Connecting to local session telemetry…"},positions:{},pinned:{},seen:{},camera:{x:24,y:24,k:1},cameraEpoch:0,fitFor:null,pointer:null,transcriptCollapsed:storedTranscriptCollapsed(),playback:{mode:"live",events:[],items:[],index:0,playing:false,speed:1,timer:null,startAt:null,endAt:null,truncated:false,gap:false},transcript:{source:null,turns:[],seen:{},query:"",follow:true,unread:0,status:"idle",session:null}}; + var state={snapshot:{schemaVersion:2,cursor:null,projects:[],sessions:[]},events:[],project:null,selected:null,node:null,browserLevel:"projects",scope:"live",historyWindow:"14d",historySnapshot:{schemaVersion:2,cursor:null,projects:[],sessions:[]},historyLoading:false,historyError:false,paused:false,pending:[],overflow:false,source:null,resyncing:false,lastAt:0,active:false,connection:{key:"connecting",text:"Connecting to local session telemetry…"},positions:{},pinned:{},seen:{},camera:{x:24,y:24,k:1},cameraEpoch:0,fitFor:null,pointer:null,transcriptCollapsed:storedTranscriptCollapsed(),playback:{mode:"live",events:[],items:[],index:0,playing:false,speed:1,timer:null,startAt:null,endAt:null,truncated:false,gap:false},transcript:{source:null,turns:[],seen:{},query:"",follow:true,unread:0,status:"idle",session:null}}; function el(id){return document.getElementById(id);} function esc(s){return String(s==null?"":s).replace(/[&<>"']/g,function(c){return{"&":"&","<":"<",">":">",'"':""","'":"'"}[c];});} function short(s,n){s=String(s||"");return s.length>n?s.slice(0,n-1)+"…":s;} @@ -46,7 +46,12 @@ export const LIVE_JS = ` function actionText(a,n,s){if(a==="tool.started")return label(n)+" started";if(a==="tool.completed")return label(n)+(s==="failed"?" failed":" completed");return{"session.started":"Session started","session.input":"New input","agent.output":"Response produced","agent.spawned":"Worker dispatched",contains:"Worker linked","agent.planned":"Agent scheduled","session.completed":"Session completed","session.failed":"Session failed"}[a]||title(a);} function activityContext(n,view){var model=String(n&&n.model||"");if(model&&model!=="unknown"&&!/^<[^>]+>$/.test(model))return model;return ago(n&&n.observedAt)||view.label;} function sessionsOf(v){var x=v&&v.snapshot?v.snapshot:v;return x&&Array.isArray(x.sessions)?x:{schemaVersion:2,cursor:null,projects:[],sessions:[]};} - function renderConnection(){var history=state.scope==="history",c=history?{key:"history",text:"History · retained local sessions"}:state.connection;el("live-state-dot").dataset.state=c.key;el("live-state-text").textContent=c.text;el("live-browser-state-dot").dataset.state=c.key;el("live-browser-state").textContent=c.text;} + // Live scope reads the SSE-fed state.snapshot; History scope reads the + // separately-fetched, date-windowed state.historySnapshot. Kept as two + // buckets (not one shared state.snapshot) so a live delta arriving while + // History is on screen can never silently overwrite the historical view. + function currentSnapshot(){return state.scope==="history"?state.historySnapshot:state.snapshot;} + function renderConnection(){var history=state.scope==="history",c=history?(state.historyLoading?{key:"connecting",text:"Loading "+state.historyWindow+" of history…"}:state.historyError?{key:"offline",text:"History unavailable · try again"}:{key:"history",text:"History · "+state.historyWindow+" · retained local sessions"}):state.connection;el("live-state-dot").dataset.state=c.key;el("live-state-text").textContent=c.text;el("live-browser-state-dot").dataset.state=c.key;el("live-browser-state").textContent=c.text;} function setConnection(k,t){state.connection={key:k,text:t};renderConnection();} function sessionKey(s){return String(s&&s.key||(hostOf(s)+":"+(s&&s.id||"")));} function presenceState(s){return s&&s.presence&&s.presence.state||"unknown";} @@ -58,13 +63,13 @@ export const LIVE_JS = ` function historicalEntityState(n){var key=n&&n.status||"unknown";if(key==="failed"||key==="cancelled"||key==="completed"||key==="blocked")return{key:key,label:statusText(key)};return{key:"historical",label:"Activity recorded"};} function displayEntityState(n,s){return historicalView(s)?historicalEntityState(n):entityState(n,s);} function displaySessionState(s){var raw=sessionState(s);if(state.scope!=="history")return raw;if(raw.key==="failed"||raw.key==="cancelled"||raw.key==="completed"||raw.key==="blocked")return raw;return{key:"historical",label:"Last active"};} - function sessionById(id){return state.snapshot.sessions.find(function(s){return sessionKey(s)===id||s.id===id;});} + function sessionById(id){return currentSnapshot().sessions.find(function(s){return sessionKey(s)===id||s.id===id;});} function projectName(s){return s&&s.project&&s.project!=="unknown"?s.project:"Project not reported";} - function rootSession(s){var seen={};while(s&&s.parentSessionId&&!seen[sessionKey(s)]){seen[sessionKey(s)]=1;var parent=state.snapshot.sessions.find(function(candidate){return candidate.host===s.host&&candidate.id===s.parentSessionId;});if(!parent)break;s=parent;}return s;} - function childSessions(root){if(!root)return[];var descendants=[],frontier=[root.id],seen={};seen[sessionKey(root)]=1;while(frontier.length){var parent=frontier.shift();state.snapshot.sessions.forEach(function(s){if(s.host===root.host&&s.parentSessionId===parent&&!seen[sessionKey(s)]){seen[sessionKey(s)]=1;descendants.push(s);frontier.push(s.id);}});}return descendants.sort(function(a,b){var ar=isLiveSession(a)?0:1,br=isLiveSession(b)?0:1;return ar-br||Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0);});} + function rootSession(s){var seen={};while(s&&s.parentSessionId&&!seen[sessionKey(s)]){seen[sessionKey(s)]=1;var parent=currentSnapshot().sessions.find(function(candidate){return candidate.host===s.host&&candidate.id===s.parentSessionId;});if(!parent)break;s=parent;}return s;} + function childSessions(root){if(!root)return[];var descendants=[],frontier=[root.id],seen={};seen[sessionKey(root)]=1;while(frontier.length){var parent=frontier.shift();currentSnapshot().sessions.forEach(function(s){if(s.host===root.host&&s.parentSessionId===parent&&!seen[sessionKey(s)]){seen[sessionKey(s)]=1;descendants.push(s);frontier.push(s.id);}});}return descendants.sort(function(a,b){var ar=isLiveSession(a)?0:1,br=isLiveSession(b)?0:1;return ar-br||Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0);});} function embeddedWorkers(root){if(!root)return[];var childIds={};childSessions(root).forEach(function(s){childIds[s.id]=1;});return(root.nodes||[]).filter(function(n){return(n.kind==="agent"||n.kind==="subagent")&&n.id!==root.id&&!childIds[n.id];}).sort(function(a,b){var aw=entityState(a,root).key==="working"?0:1,bw=entityState(b,root).key==="working"?0:1;return aw-bw||String(label(a)).localeCompare(String(label(b)));});} function isNavigationRoot(s){return!!s&&(s.navigationRoot===true||(s.navigationRoot!==false&&rootSession(s)===s));} - function visibleSessions(){return state.snapshot.sessions.filter(function(s){return isNavigationRoot(s)&&String(s.projectKey||projectName(s))===state.project&&(state.scope==="live"?isLiveSession(s):!isLiveSession(s));}).sort(function(a,b){return Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0);});} + function visibleSessions(){return currentSnapshot().sessions.filter(function(s){return isNavigationRoot(s)&&String(s.projectKey||projectName(s))===state.project&&(state.scope==="live"?isLiveSession(s):!isLiveSession(s));}).sort(function(a,b){return Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0);});} function receiveSnapshot(v,reset){state.snapshot=sessionsOf(v);state.lastAt=Date.now();if(reset){state.events=[];state.positions={};state.seen={};state.node=null;state.fitFor=null;}var projects=projectCatalog();if(!projects.some(function(p){return p.id===state.project;}))state.project=projects[0]&&projects[0].id||null;if(!sessionById(state.selected)||String(sessionById(state.selected).projectKey||projectName(sessionById(state.selected)))!==state.project){var list=visibleSessions();selectSession(list[0]&&sessionKey(list[0]),true);}render();} function applyEvent(ev){ if(!ev||!ev.sessionId||!ev.actor||!ev.actor.id)return;if(ev.eventId&&state.seen[ev.eventId])return; @@ -78,6 +83,11 @@ export const LIVE_JS = ` function receive(k,d){if(state.paused){if(state.pending.length"+esc(label(value))+""+esc(kindName(value.kind)+" · "+view.label)+(ownerNode?"
Owned by "+esc(label(ownerNode)):"")+"
"+esc(hostName(value)+" · "+inferenceProviderName(value))+"
Evidence: "+esc(value.confidence||"not reported");tip.hidden=false;var w=Math.min(300,window.innerWidth-24),left=Math.max(12,Math.min(window.innerWidth-w-12,r.right+10)),top=Math.max(12,Math.min(window.innerHeight-tip.offsetHeight-12,r.top));tip.style.width=w+"px";tip.style.left=left+"px";tip.style.top=top+"px";target.setAttribute("aria-describedby","live-tooltip");} function hideTooltip(target){if(target&&document.activeElement===target)return;el("live-tooltip").hidden=true;if(target)target.removeAttribute("aria-describedby");} function showRowTooltip(target){if(!target)return;var tip=el("live-tooltip"),r=target.getBoundingClientRect(),heading=target.querySelector(".live-session-main span,.live-project span"),description=target.getAttribute("aria-label")||target.title||target.textContent;tip.innerHTML=""+esc(heading&&heading.textContent||"Session")+""+esc(description);tip.hidden=false;var w=Math.min(300,window.innerWidth-24),left=Math.max(12,Math.min(window.innerWidth-w-12,r.right+10)),top=Math.max(12,Math.min(window.innerHeight-tip.offsetHeight-12,r.top));tip.style.width=w+"px";tip.style.left=left+"px";tip.style.top=top+"px";target.setAttribute("aria-describedby","live-tooltip");} - function projectCatalog(){var given=Array.isArray(state.snapshot.projects)?state.snapshot.projects:[],by={};state.snapshot.sessions.forEach(function(s){if(!s.project||s.project==="unknown")return;var id=String(s.projectKey||projectName(s)),p=by[id]||(by[id]={id:id,name:projectName(s),sessionCount:0,liveCount:0,historicalCount:0,liveChildCount:0,historicalChildCount:0,updatedAt:s.updatedAt}),live=isLiveSession(s);if(isNavigationRoot(s)){p.sessionCount++;if(live)p.liveCount++;else p.historicalCount++;}else if(live)p.liveChildCount++;else p.historicalChildCount++;if(Date.parse(s.updatedAt||0)>Date.parse(p.updatedAt||0))p.updatedAt=s.updatedAt;});given.forEach(function(p){var name=p.label||p.name||p.project;if(!name||name==="unknown")return;var id=String(p.id||p.key||p.projectKey||name),derived=by[id]||{};if(Object.keys(derived).length)by[id]=Object.assign({},p,derived,{id:id,name:name||derived.name});});return Object.keys(by).map(function(k){return by[k];}).sort(function(a,b){return Number(b.liveCount||0)-Number(a.liveCount||0)||Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0)||a.name.localeCompare(b.name);});} + function projectCatalog(){var snap=currentSnapshot(),given=Array.isArray(snap.projects)?snap.projects:[],by={};snap.sessions.forEach(function(s){if(!s.project||s.project==="unknown")return;var id=String(s.projectKey||projectName(s)),p=by[id]||(by[id]={id:id,name:projectName(s),sessionCount:0,liveCount:0,historicalCount:0,liveChildCount:0,historicalChildCount:0,updatedAt:s.updatedAt}),live=isLiveSession(s);if(isNavigationRoot(s)){p.sessionCount++;if(live)p.liveCount++;else p.historicalCount++;}else if(live)p.liveChildCount++;else p.historicalChildCount++;if(Date.parse(s.updatedAt||0)>Date.parse(p.updatedAt||0))p.updatedAt=s.updatedAt;});given.forEach(function(p){var name=p.label||p.name||p.project;if(!name||name==="unknown")return;var id=String(p.id||p.key||p.projectKey||name),derived=by[id]||{};if(Object.keys(derived).length)by[id]=Object.assign({},p,derived,{id:id,name:name||derived.name});});return Object.keys(by).map(function(k){return by[k];}).sort(function(a,b){return Number(b.liveCount||0)-Number(a.liveCount||0)||Date.parse(b.updatedAt||0)-Date.parse(a.updatedAt||0)||a.name.localeCompare(b.name);});} function duration(s){var start=s.startedAt||s.createdAt,end=s.endedAt||s.completedAt||((s.status==="completed"||s.status==="failed"||s.status==="cancelled")&&s.updatedAt),a=Date.parse(start||""),b=Date.parse(end||""),ms=b-a;if(!Number.isFinite(a)||!Number.isFinite(b)||ms<=0)return ago(s.updatedAt)||"activity time unavailable";var m=Math.max(1,Math.round(ms/60000));return m<60?m+"m":Math.floor(m/60)+"h "+m%60+"m";} function sessionMarkup(s,child){var live=state.scope==="live"&&isLiveSession(s),key=sessionKey(s),view=displaySessionState(s),identity=hostName(s)+" · "+inferenceProviderName(s),name=s.title||s.summary||(child?"Worker thread":hostName(s)+" session"),description=name+". "+view.label+". "+duration(s)+". "+workspaceDescription(s);return'";} function workerMarkup(s,n){var view=displayEntityState(n,s),worker=n&&n.host?n:s,description=(label(n)||"Worker")+". Worker view. "+view.label+". "+workspaceDescription(s);return'";} - function renderSessions(){var allProjects=projectCatalog(),projects=allProjects.filter(function(p){return Number(state.scope==="live"?p.liveCount:p.historicalCount)>0;}),list=visibleSessions(),current=projects.find(function(p){return p.id===state.project;}),selected=sessionById(state.selected),browser=el("live-browser"),scopedRoots=state.snapshot.sessions.filter(function(s){return isNavigationRoot(s)&&(state.scope==="live"?isLiveSession(s):!isLiveSession(s));});if(!current){state.project=projects[0]&&projects[0].id||null;current=projects[0];list=visibleSessions();}browser.dataset.level=state.browserLevel;el("live-view-summary").textContent=state.scope==="live"?(scopedRoots.length?scopedRoots.length+" active session"+(scopedRoots.length===1?"":"s")+" across "+projects.length+" project"+(projects.length===1?"":"s"):"No live sessions across 0 projects"):(scopedRoots.length+" historical session"+(scopedRoots.length===1?"":"s")+" across "+projects.length+" project"+(projects.length===1?"":"s"));el("live-browser-kicker").textContent=state.browserLevel==="sessions"?"PROJECT":state.scope==="live"?"LIVE WORKSPACES":"HISTORY";el("live-browser-title").textContent=state.browserLevel==="sessions"&¤t?short(current.name,26):"Projects";el("live-project-count").textContent=String(projects.length);el("live-project-list").innerHTML=projects.map(function(p){var sessions=Number(state.scope==="live"?p.liveCount:p.historicalCount),workers=Number(state.scope==="live"?p.liveChildCount:p.historicalChildCount),description=p.name+". "+sessions+" "+(state.scope==="live"?"live":"historical")+" sessions"+(workers?" and "+workers+" workers":"")+". "+(ago(p.updatedAt)||"Activity time unavailable");return'";}).join("")||'
'+(state.scope==="live"?"No projects have a live session.":"No retained project history found.")+"
";var selectedRoot=rootSession(selected);if(!selectedRoot||!list.some(function(s){return sessionKey(s)===sessionKey(selectedRoot);})){selectSession(list[0]&&sessionKey(list[0]),true);list=visibleSessions();selected=sessionById(state.selected);selectedRoot=rootSession(selected);}el("live-session-heading").textContent=current?short(current.name,36):"Sessions";el("live-count").textContent=String(list.length);el("live-session-list").innerHTML=list.map(function(s){var expanded=sessionKey(s)===sessionKey(selectedRoot),children=expanded?childSessions(s).filter(function(child){return state.scope==="live"?isLiveSession(child):!isLiveSession(child);}):[],workers=expanded?embeddedWorkers(s).filter(function(worker){var working=entityState(worker,s).key==="working";return state.scope==="live"?working:!working;}):[],nested=children.map(function(child){return sessionMarkup(child,true);}).concat(workers.map(function(worker){return workerMarkup(s,worker);}));return'
'+sessionMarkup(s,false)+(nested.length?'
'+nested.join("")+"
":"")+"
";}).join("")||'
No '+(state.scope==="live"?"live":"historical")+' sessions for this project.
';el("live-session-context-identity").textContent=selected?hostName(selected)+" · "+inferenceProviderName(selected):"—";el("live-session-context-project").textContent=selected?projectName(selected):"Choose a session";var selectedState=displaySessionState(selected);el("live-session-context-status").textContent=selected?selectedState.label+" · "+duration(selected):"Waiting for local evidence";} + function renderSessions(){var allProjects=projectCatalog(),projects=allProjects.filter(function(p){return Number(state.scope==="live"?p.liveCount:p.historicalCount)>0;}),list=visibleSessions(),current=projects.find(function(p){return p.id===state.project;}),selected=sessionById(state.selected),browser=el("live-browser"),scopedRoots=currentSnapshot().sessions.filter(function(s){return isNavigationRoot(s)&&(state.scope==="live"?isLiveSession(s):!isLiveSession(s));});if(!current){state.project=projects[0]&&projects[0].id||null;current=projects[0];list=visibleSessions();}browser.dataset.level=state.browserLevel;el("live-view-summary").textContent=state.scope==="live"?(scopedRoots.length?scopedRoots.length+" active session"+(scopedRoots.length===1?"":"s")+" across "+projects.length+" project"+(projects.length===1?"":"s"):"No live sessions across 0 projects"):(scopedRoots.length+" historical session"+(scopedRoots.length===1?"":"s")+" across "+projects.length+" project"+(projects.length===1?"":"s"));el("live-browser-kicker").textContent=state.browserLevel==="sessions"?"PROJECT":state.scope==="live"?"LIVE WORKSPACES":"HISTORY";el("live-browser-title").textContent=state.browserLevel==="sessions"&¤t?short(current.name,26):"Projects";el("live-project-count").textContent=String(projects.length);el("live-project-list").innerHTML=projects.map(function(p){var sessions=Number(state.scope==="live"?p.liveCount:p.historicalCount),workers=Number(state.scope==="live"?p.liveChildCount:p.historicalChildCount),description=p.name+". "+sessions+" "+(state.scope==="live"?"live":"historical")+" sessions"+(workers?" and "+workers+" workers":"")+". "+(ago(p.updatedAt)||"Activity time unavailable");return'";}).join("")||'
'+(state.scope==="live"?"No projects have a live session.":"No retained project history found.")+"
";var selectedRoot=rootSession(selected);if(!selectedRoot||!list.some(function(s){return sessionKey(s)===sessionKey(selectedRoot);})){selectSession(list[0]&&sessionKey(list[0]),true);list=visibleSessions();selected=sessionById(state.selected);selectedRoot=rootSession(selected);}el("live-session-heading").textContent=current?short(current.name,36):"Sessions";el("live-count").textContent=String(list.length);el("live-session-list").innerHTML=list.map(function(s){var expanded=sessionKey(s)===sessionKey(selectedRoot),children=expanded?childSessions(s).filter(function(child){return state.scope==="live"?isLiveSession(child):!isLiveSession(child);}):[],workers=expanded?embeddedWorkers(s).filter(function(worker){var working=entityState(worker,s).key==="working";return state.scope==="live"?working:!working;}):[],nested=children.map(function(child){return sessionMarkup(child,true);}).concat(workers.map(function(worker){return workerMarkup(s,worker);}));return'
'+sessionMarkup(s,false)+(nested.length?'
'+nested.join("")+"
":"")+"
";}).join("")||'
No '+(state.scope==="live"?"live":"historical")+' sessions for this project.
';el("live-session-context-identity").textContent=selected?hostName(selected)+" · "+inferenceProviderName(selected):"—";el("live-session-context-project").textContent=selected?projectName(selected):"Choose a session";var selectedState=displaySessionState(selected);el("live-session-context-status").textContent=selected?selectedState.label+" · "+duration(selected):"Waiting for local evidence";} function renderHealth(){var h=state.snapshot.health||{},bad=0;el("live-health").innerHTML=Object.keys(h).sort().map(function(n){var v=h[n]||{};if(v.status==="error")bad++;return''+esc(n)+' · '+esc(v.status||"unknown")+" · "+Number(v.files||0)+" files · "+Number(v.events||0)+" events · "+Number(v.errors||0)+" errors";}).join("");el("live-health-toggle").textContent=bad?bad+" source issue"+(bad===1?"":"s"):"Sources";} - function render(){el("panel-observability").dataset.scope=state.scope;document.querySelectorAll("[data-live-scope]").forEach(function(item){item.setAttribute("aria-selected",String(item.dataset.liveScope===state.scope));});el("observability-title").textContent=state.scope==="live"?"Live agent activity":"Session history";el("live-sub").textContent=state.scope==="live"?"Follow current sessions, agents, tools, and supported evidence as work happens.":"Inspect retained, inert evidence and deterministic playback from completed work.";el("live-transcript-kicker").textContent=state.scope==="live"?"LIVE EVIDENCE":"SESSION EVIDENCE";el("live-transcript-title").textContent=state.scope==="live"?"Session stream":"Recorded session";el("live-transcript-search").placeholder=state.scope==="live"?"Search this stream…":"Search retained evidence…";el("live-transcript-list").setAttribute("aria-live",state.scope==="live"?"polite":"off");renderConnection();renderSessions();renderHealth();el("live-cursor").textContent=state.snapshot.cursor||"";var source=sessionById(state.selected),s=state.playback.mode==="review"&&state.playback.session||source;renderGraph(s);renderPlayback();if(s&&state.fitFor!==sessionKey(s)){state.fitFor=sessionKey(s);(window.requestAnimationFrame||setTimeout)(function(){fit();});}} + function render(){el("panel-observability").dataset.scope=state.scope;document.querySelectorAll("[data-live-scope]").forEach(function(item){item.setAttribute("aria-selected",String(item.dataset.liveScope===state.scope));});var windowGroup=el("observability-window");if(windowGroup)windowGroup.hidden=state.scope!=="history";el("observability-title").textContent=state.scope==="live"?"Live agent activity":"Session history";el("live-sub").textContent=state.scope==="live"?"Follow current sessions, agents, tools, and supported evidence as work happens.":"Inspect retained, inert evidence and deterministic playback from completed work.";el("live-transcript-kicker").textContent=state.scope==="live"?"LIVE EVIDENCE":"SESSION EVIDENCE";el("live-transcript-title").textContent=state.scope==="live"?"Session stream":"Recorded session";el("live-transcript-search").placeholder=state.scope==="live"?"Search this stream…":"Search retained evidence…";el("live-transcript-list").setAttribute("aria-live",state.scope==="live"?"polite":"off");renderConnection();renderSessions();renderHealth();el("live-cursor").textContent=currentSnapshot().cursor||"";var source=sessionById(state.selected),s=state.playback.mode==="review"&&state.playback.session||source;renderGraph(s);renderPlayback();if(s&&state.fitFor!==sessionKey(s)){state.fitFor=sessionKey(s);(window.requestAnimationFrame||setTimeout)(function(){fit();});}} function closeTranscript(){if(state.transcript.source)state.transcript.source.close();state.transcript.source=null;state.transcript.status="idle";} function transcriptStatus(k,t){state.transcript.status=k;var e=el("live-transcript-state");e.dataset.state=k;e.textContent=t;} function transcriptItems(v){if(Array.isArray(v))return v;if(v&&v.snapshot&&Array.isArray(v.snapshot.events))return v.snapshot.events;if(v&&Array.isArray(v.events))return v.events;if(v&&Array.isArray(v.items))return v.items;if(v&&Array.isArray(v.turns))return v.turns;if(v&&v.item)return[v.item];if(v&&v.turn)return[v.turn];return v&&typeof v==="object"?[v]:[];} @@ -137,13 +147,14 @@ export const LIVE_JS = ` function selectSession(id,open){var s=sessionById(id),key=s?sessionKey(s):id;if(state.selected===key&&state.transcript.session===key)return;stopPlayback();state.selected=key||null;state.node=null;state.fitFor=null;state.camera={x:24,y:24,k:1};if(open!==false){if(isLiveSession(s)){state.playback.mode="live";openTranscript(s);}else if(s)loadPlayback(s);}} function selectNode(id){state.node=id||null;render();renderTranscript();var details=el("live-selection");if(id)details.open=true;var found=id&&tooltipValue(id).value;el("live-interaction-status").textContent=found?label(found)+" selected. Details opened.":"Showing all actors.";} function setTranscriptCollapsed(collapsed,persist){collapsed=!!collapsed;state.transcriptCollapsed=collapsed;var workspace=el("live-workspace"),panel=el("live-transcript-panel"),body=el("live-transcript-body"),button=el("live-transcript-toggle"),expanded=!collapsed;if(workspace)workspace.dataset.transcriptCollapsed=String(collapsed);if(panel)panel.dataset.collapsed=String(collapsed);if(body)body.hidden=collapsed;if(button){button.setAttribute("aria-expanded",String(expanded));button.setAttribute("aria-label",expanded?"Collapse Session stream":"Expand Session stream");button.title=expanded?"Collapse Session stream and expand Agent activity":"Expand Session stream";}if(persist!==false){try{localStorage.setItem(TRANSCRIPT_COLLAPSE_KEY,String(collapsed));}catch(_){}}if(el("live-interaction-status"))el("live-interaction-status").textContent=collapsed?"Session stream collapsed. Agent activity has more room.":"Session stream expanded.";} - function setScope(scope,sync){if(scope!=="live"&&scope!=="history")return false;if(scope===state.scope){render();return true;}closeTranscript();stopPlayback();state.scope=scope;state.browserLevel="projects";state.project=null;state.selected=null;state.node=null;state.transcript.follow=scope==="live";state.transcript.unread=0;state.playback.mode="live";render();if(sync!==false&&window.AKDashboardSyncHash)window.AKDashboardSyncHash();return true;} + function setScope(scope,sync){if(scope!=="live"&&scope!=="history")return false;if(scope===state.scope){render();return true;}closeTranscript();stopPlayback();state.scope=scope;state.browserLevel="projects";state.project=null;state.selected=null;state.node=null;state.transcript.follow=scope==="live";state.transcript.unread=0;state.playback.mode="live";if(scope==="history")fetchHistory();else render();if(sync!==false&&window.AKDashboardSyncHash)window.AKDashboardSyncHash();return true;} function bind(){ setTranscriptCollapsed(state.transcriptCollapsed,false);el("live-transcript-toggle").addEventListener("click",function(){setTranscriptCollapsed(!state.transcriptCollapsed,true);}); el("live-pause").addEventListener("click",function(){state.paused=!state.paused;this.setAttribute("aria-pressed",String(state.paused));this.innerHTML=state.paused?"▶ Resume":"⏸ Pause";el("live-canvas").dataset.paused=String(state.paused);var svg=el("live-graph");if(state.paused&&svg.pauseAnimations)svg.pauseAnimations();else if(!state.paused&&svg.unpauseAnimations)svg.unpauseAnimations();if(!state.paused){var q=state.pending.slice();state.pending=[];if(state.overflow){state.overflow=false;connect();}else q.forEach(function(p){receive(p.kind,p.data);});}}); el("live-health-toggle").addEventListener("click",function(){var h=el("live-health"),open=h.hidden;h.hidden=!open;this.setAttribute("aria-expanded",String(open));if(open){var r=this.getBoundingClientRect(),w=Math.min(360,window.innerWidth-24);h.style.width=w+"px";h.style.left=Math.max(12,Math.min(window.innerWidth-w-12,r.right-w))+"px";h.style.top=Math.min(window.innerHeight-h.offsetHeight-12,r.bottom+8)+"px";}}); el("live-project-list").addEventListener("click",function(e){var b=e.target.closest("[data-project]");if(!b)return;state.project=b.dataset.project;state.browserLevel="sessions";var list=visibleSessions();selectSession(list[0]&&sessionKey(list[0]));render();(el("live-session-list").querySelector("[data-session]")||el("live-browser-back")).focus();}); var scopeTabs=Array.from(document.querySelectorAll("[data-live-scope]"));function activateScope(button){if(button)setScope(button.dataset.liveScope,true);}scopeTabs.forEach(function(button){button.addEventListener("click",function(){activateScope(this);});});el("live-scope-tabs").addEventListener("keydown",function(e){if(!/^(ArrowLeft|ArrowRight|Home|End)$/.test(e.key))return;var current=Math.max(0,scopeTabs.indexOf(document.activeElement)),next=e.key==="Home"?0:e.key==="End"?scopeTabs.length-1:(current+(e.key==="ArrowRight"?1:-1)+scopeTabs.length)%scopeTabs.length;e.preventDefault();scopeTabs[next].focus();activateScope(scopeTabs[next]);}); + var windowChips=el("observability-window");if(windowChips)windowChips.addEventListener("click",function(e){var b=e.target.closest("[data-history-window]");if(!b)return;var token=b.dataset.historyWindow;if(token===state.historyWindow)return;state.historyWindow=token;var all=windowChips.querySelectorAll("[data-history-window]");for(var i=0;iLive + diff --git a/src/lib/live/live-sessions-service.mjs b/src/lib/live/live-sessions-service.mjs index 70a3998..9e0901e 100644 --- a/src/lib/live/live-sessions-service.mjs +++ b/src/lib/live/live-sessions-service.mjs @@ -19,6 +19,14 @@ import { bootstrapRecords, codexTranscriptId, discoverJsonl, } from './native-transcript-discovery.mjs'; +// historySnapshot() is a one-shot on-demand scan, not a continuously-tailed +// live feed, so it can afford limits well above the live path's maxFiles(256) +// /maxSessions(100) defaults — those stay small purely to keep the always-on +// tailer set cheap. discoverJsonl's own hard 4096-file safety cap is the real +// backstop for the "all time" window on a machine with a large corpus. +const HISTORY_MAX_FILES = 2048; +const HISTORY_MAX_SESSIONS = 1000; + /** * Coordinates bounded transcript tailers into one privacy-safe live projection. * Transcript contents exist only for the duration of adapter calls. @@ -211,7 +219,11 @@ export class LiveSessionsService { this.#mark(context.adapter, { files: (this.#health.get(context.adapter)?.files ?? 0) + 1 }); } - #record(record, context, file) { + /** Pure record → LiveEvent[] transformation, shared by the live tailer + * (#record, below) and historySnapshot()'s one-shot scan. Mutates `context` + * in place (identity/provider/model learned as records stream by) but + * touches no instance state beyond the read-only #claudeProvider cache. */ + #buildEvents(record, context, file) { const explicitCwd = record?.cwd ?? (['session_meta', 'turn_context'].includes(record?.type) ? record.payload?.cwd : null); if (explicitCwd) { @@ -239,24 +251,74 @@ export class LiveSessionsService { const common = { ...context, artifact: file, observedAt: this.#options.now(), }; - let events; - if (context.adapter === 'claude') { - events = adaptClaudeRecord(record, common); - } else if (context.adapter === 'codex') { + if (context.adapter === 'claude') return adaptClaudeRecord(record, common); + if (context.adapter === 'codex') { if (record?.type === 'session_meta' && record.payload?.id) { context.sessionId = record.payload.id; context.meta = record.payload; } - events = adaptCodexRecord(record, common); - } else { - events = adaptStructuredEvent(record, { - ...common, artifact: path.basename(file), - surface: context.surface, adapter: `${context.surface}-jsonl`, - }); + return adaptCodexRecord(record, common); } + return adaptStructuredEvent(record, { + ...common, artifact: path.basename(file), + surface: context.surface, adapter: `${context.surface}-jsonl`, + }); + } + + #record(record, context, file) { + const events = this.#buildEvents(record, context, file); for (const event of events) this.#publish(event, context.adapter); } + /** + * On-demand, date-windowed scan for the Observability "History" browser. + * Independent of the live tailer: builds and discards its own projection, + * so it never touches #projection/#workspaceStore/#health and can never + * evict or otherwise disturb the live in-memory state. Reuses the exact + * same discovery/bootstrap/adapter pipeline as the live path so a session + * renders identically whichever scope produced it. + * @param {{ sinceMs?: number|null }} [options] sinceMs is an epoch-ms + * cutoff on file mtime; omit/null scans "all time". + * @returns {ReturnType} + */ + historySnapshot({ sinceMs = null } = {}) { + const claude = discoverJsonl(this.#options.roots.claude, { + maxDepth: 3, maxFiles: HISTORY_MAX_FILES, sinceMs, accept: () => true, + }); + const codex = discoverJsonl(this.#options.roots.codex, { + maxDepth: 4, maxFiles: HISTORY_MAX_FILES, sinceMs, accept: (name) => name.startsWith('rollout-'), + }); + let projection = emptyLiveProjection(); + const ingest = (file, adapter, context) => { + for (const record of bootstrapRecords(file, adapter)) { + for (const event of this.#buildEvents(record, context, file)) { + projection = reduceLiveEvent(projection, event, { + maxSessions: HISTORY_MAX_SESSIONS, maxNodesPerSession: this.#options.maxNodesPerSession, + }); + } + } + }; + for (const file of claude) { + ingest(file, 'claude', { adapter: 'claude', sessionId: path.basename(file, '.jsonl'), project: 'unknown' }); + } + for (const file of codex) { + ingest(file, 'codex', { adapter: 'codex', sessionId: codexTranscriptId(file), meta: {} }); + } + // A one-shot scan never observes the process ending, so the reducer's + // last-known state for an unterminated session defaults to lifecycle + // 'active'/status 'running' — correct for the live tailer (which sweeps + // continuously as time passes) but wrong here: it would read as a LIVE + // session and get excluded from the History browser's session list + // (which explicitly filters OUT anything isLiveSession() still calls + // live). All-zero windows force every non-terminal session to read as + // stale immediately, which is the only honest answer for retained + // evidence being browsed well after the fact. + projection = sweepLiveProjection(projection, { + now: this.#options.now(), quiescentMs: 0, expiryMs: 0, pendingExpiryMs: 0, + }); + return serializeLiveProjection(projection); + } + /** Configuration reads are per-project, so memoize by session cwd. */ #claudeProvider(cwd) { if (!this.#claudeProviders.has(cwd)) { diff --git a/src/lib/live/native-transcript-discovery.mjs b/src/lib/live/native-transcript-discovery.mjs index 68a58a6..3cfdcf0 100644 --- a/src/lib/live/native-transcript-discovery.mjs +++ b/src/lib/live/native-transcript-discovery.mjs @@ -5,7 +5,15 @@ const safeEntries = (dir) => { try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; } }; -export function discoverJsonl(root, { maxDepth, maxFiles, accept }) { +/** + * @param {string} root + * @param {{ maxDepth: number, maxFiles: number, accept: (name: string) => boolean, + * sinceMs?: number|null }} options sinceMs, when given, drops any file whose + * mtime is older than that epoch-ms cutoff — used for date-windowed history scans. + * Omit/null (the default) preserves the unfiltered recency-only behavior every + * existing caller (the live tailer, project-discovery) relies on. + */ +export function discoverJsonl(root, { maxDepth, maxFiles, accept, sinceMs = null }) { const found = []; const visit = (dir, depth) => { if (depth > maxDepth || found.length >= 4096) return; @@ -16,6 +24,7 @@ export function discoverJsonl(root, { maxDepth, maxFiles, accept }) { else if (entry.isFile() && entry.name.endsWith('.jsonl') && accept(entry.name)) { let mtimeMs = 0; try { mtimeMs = fs.statSync(file).mtimeMs; } catch { /* no ordering evidence */ } + if (sinceMs != null && mtimeMs < sinceMs) continue; found.push({ file, mtimeMs }); } } diff --git a/tests/dashboard.test.cjs b/tests/dashboard.test.cjs index 74b84a5..bfd3c7e 100644 --- a/tests/dashboard.test.cjs +++ b/tests/dashboard.test.cjs @@ -982,6 +982,45 @@ async function main() { assert(liveCalls.start === 1, 'live collector must start exactly once'); }); + await test('GET /api/live/history against a live service without historySnapshot → 501', async () => { + const r = await get(liveSrv.url + 'api/live/history', liveSrv.token); + assert(r.status === 501, 'expected 501, got ' + r.status); + }); + + await test('GET /api/live/history resolves window→sinceMs and scrubs the same as /api/live', async () => { + const historyCalls = []; + live.historySnapshot = (opts) => { + historyCalls.push(opts); + return { + schemaVersion: 1, cursor: null, + sessions: [{ + id: 'h1', project: 'agentic-kit', projectKey: 'project:test', + nodes: [{ + id: 'hn1', source: { artifact: '/Users/private/.claude/projects/history-session.jsonl' }, + response: 'PRIVATE HISTORICAL RESPONSE', + }], + edges: [], + }], + projects: [{ id: 'project:test', label: 'agentic-kit' }], + }; + }; + const noWindow = await get(liveSrv.url + 'api/live/history', liveSrv.token); + assert(noWindow.status === 200, 'expected 200, got ' + noWindow.status); + assert(!noWindow.body.includes('PRIVATE HISTORICAL RESPONSE'), 'history snapshot must be scrubbed too'); + assert(!noWindow.body.includes('/Users/private'), 'artifact paths must be reduced to a leaf, not the absolute path'); + assert(noWindow.body.includes('history-session.jsonl'), 'the artifact leaf name itself is allowed through'); + const fourteenDayMs = 14 * 24 * 60 * 60 * 1000; + assert(Math.abs(Date.now() - historyCalls[0].sinceMs - fourteenDayMs) < 5000, + 'missing ?window must default to a 14-day cutoff, same as clampDays'); + + await get(liveSrv.url + 'api/live/history?window=1d', liveSrv.token); + const oneDayMs = 24 * 60 * 60 * 1000; + assert(Math.abs(Date.now() - historyCalls[1].sinceMs - oneDayMs) < 5000, '?window=1d must resolve to a 1-day cutoff'); + + await get(liveSrv.url + 'api/live/history?window=all', liveSrv.token); + assert(historyCalls[2].sinceMs === null, '?window=all must scan without a cutoff'); + }); + await test('GET /api/live/events with no token → 401 before any subscribe', async () => { const before = liveCalls.subscribe; const r = await getRaw(liveSrv.port, '/api/live/events'); @@ -1408,7 +1447,7 @@ async function main() { // is the suite where it matters most — the traversal-guard and credential- // leak tests live here and were the reviewer's cited example of a block // that could silently vanish with the old harness never noticing. - const EXPECTED = 63; + const EXPECTED = 65; if (passed + failed !== EXPECTED) { console.error(`\nPLAN MISMATCH: expected ${EXPECTED} tests, ran ${passed + failed}`); process.exit(1); diff --git a/tests/kit/live-service.test.mjs b/tests/kit/live-service.test.mjs index 6e970b7..97349de 100644 --- a/tests/kit/live-service.test.mjs +++ b/tests/kit/live-service.test.mjs @@ -630,3 +630,46 @@ test('ledger edges retain their repository after bounded projection eviction', ( assert.equal(parent.project, 'agentic-kit'); assert.ok(!service.snapshot().projects.some((project) => project.label === 'unknown')); }); + +test('historySnapshot() date-windows a one-shot scan without disturbing the live tailer', (t) => { + const sb = sandbox(); + const now = Date.parse('2026-08-06T12:00:00Z'); + const recent = path.join(sb.claude, 'recent.jsonl'); + const old = path.join(sb.claude, 'old.jsonl'); + fs.writeFileSync(recent, line({ + type: 'user', sessionId: 'recent', timestamp: '2026-08-06T11:00:00Z', + cwd: '/Users/private-user/work/visible-project', + message: { role: 'user', content: [{ type: 'text', text: 'hi' }] }, + })); + fs.writeFileSync(old, line({ + type: 'user', sessionId: 'old', timestamp: '2026-01-01T11:00:00Z', + cwd: '/Users/private-user/work/other-project', + message: { role: 'user', content: [{ type: 'text', text: 'hi' }] }, + })); + // discoverJsonl sorts/filters by mtime, not the record's own timestamp — + // stamp the file itself so the sinceMs cutoff below has something real to bite on. + const oldMs = Date.parse('2026-01-01T11:00:00Z') / 1000; + fs.utimesSync(old, oldMs, oldMs); + + const service = new LiveSessionsService({ + roots: sb.roots, intervalMs: 10, readCodexState: () => null, now: () => new Date(now).toISOString(), + }); + t.after(() => service.close()); + + // A 1-day window sees only the recent file. + const windowed = service.historySnapshot({ sinceMs: now - 86_400_000 }); + assert.deepEqual(windowed.sessions.map((s) => s.id), ['recent']); + // "all time" (sinceMs omitted) sees both. + const all = service.historySnapshot(); + assert.deepEqual(all.sessions.map((s) => s.id).sort(), ['old', 'recent']); + + // A one-shot scan of an unterminated session must never read as "live" — + // it would otherwise vanish from the History browser's session list, which + // explicitly filters OUT anything still reading as live. + assert.equal(windowed.projects[0].liveCount, 0); + assert.equal(windowed.sessions[0].activity.state, 'idle'); + + // The live tailer's own state is untouched: it was never start()ed, so its + // projection is still empty — historySnapshot() must not have populated it. + assert.deepEqual(service.snapshot().sessions, []); +}); diff --git a/tests/ui/dashboard-ui.mjs b/tests/ui/dashboard-ui.mjs index 008b198..0f0a702 100644 --- a/tests/ui/dashboard-ui.mjs +++ b/tests/ui/dashboard-ui.mjs @@ -348,6 +348,11 @@ const LIVE_SNAPSHOT = { const LIVE_STUB = { start: async () => {}, snapshot: async () => LIVE_SNAPSHOT, + // The fixture is static, so History's ?window= is a no-op here — the real + // LiveSessionsService.historySnapshot() date-windowing is covered by + // tests/kit/live-service.test.mjs; this stub only needs to exist so the + // Observability → History tab has something to render end-to-end. + historySnapshot: async () => LIVE_SNAPSHOT, replay: async () => ({ reset: false, events: [] }), subscribe: () => () => {}, close: async () => {}, @@ -418,10 +423,10 @@ async function main() { }); page.on('pageerror', (e) => consoleErrors.push(String(e.message))); page.on('requestfailed', (r) => { - // Leaving Live deliberately closes EventSource. Chromium reports that - // client-side teardown as ERR_ABORTED even though it is the expected, - // leak-preventing lifecycle behavior. - if (/\/api\/live\/(?:events|transcripts\/[^/]+\/[^/]+\/events)(?:\?.*)?$/.test(r.url()) + // Leaving Live (or the Intelligence view) deliberately closes its + // EventSource. Chromium reports that client-side teardown as ERR_ABORTED + // even though it is the expected, leak-preventing lifecycle behavior. + if (/\/api\/live\/(?:events|intelligence|transcripts\/[^/]+\/[^/]+\/events)(?:\?.*)?$/.test(r.url()) && r.failure()?.errorText === 'net::ERR_ABORTED') return; failedRequests.push(`${r.url()} — ${r.failure()?.errorText}`); }); From 0bd5c12d1cd42ec01039d9c1fe90eb986cd155a5 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 13:13:33 -0700 Subject: [PATCH 03/19] =?UTF-8?q?docs:=20propose=20the=20System=20area=20?= =?UTF-8?q?=E2=80=94=20ADR-0025,=20machine-footprint=20domain,=20design=20?= =?UTF-8?q?mock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drafts for review; nothing implemented yet. - ADR-0025 (Proposed): a Machine footprint bounded context and a fourth System primary area (Summary / Storage / Runtime / Catalog / Projects), tiered honest collection with a persisted asOf snapshot, the initial metric taxonomy (install, runtime, storage, catalog, per-project LOC/disk, git-remote links), GET /api/system + ak footprint delivery, advisory-only reclaimables, and a documented absolute-path exception - docs/ddd/machine-footprint.md: purpose, boundaries against Usage / Observability / Project intelligence / Integration management, the FootprintSnapshot model, measurement semantics, 12 invariants, and proposed ubiquitous-language terms - docs/assets/system-tab-mock.html: self-contained both-theme mock of the System area on the dashboard's own tokens, every card annotated with its chart-form rationale; illustrative data only --- docs/adr/0025-machine-footprint-metrics.md | 244 ++++++++ docs/assets/system-tab-mock.html | 643 +++++++++++++++++++++ docs/ddd/machine-footprint.md | 236 ++++++++ 3 files changed, 1123 insertions(+) create mode 100644 docs/adr/0025-machine-footprint-metrics.md create mode 100644 docs/assets/system-tab-mock.html create mode 100644 docs/ddd/machine-footprint.md diff --git a/docs/adr/0025-machine-footprint-metrics.md b/docs/adr/0025-machine-footprint-metrics.md new file mode 100644 index 0000000..3845e98 --- /dev/null +++ b/docs/adr/0025-machine-footprint-metrics.md @@ -0,0 +1,244 @@ +# ADR-0025 — Machine footprint: infrastructure metrics for install, runtime, storage, and catalog + +- **Status:** Proposed (draft for review — no implementation exists yet) +- **Date:** 2026-08-06 +- **Deciders:** agentic-kit maintainers +- **Related:** [ADR-0005](0005-dashboard-in-page-routing-reveal.md), + [ADR-0007](0007-maintainer-admin-local-telemetry.md), + [ADR-0009](0009-usage-scorecard-local-transcript-analytics.md), + [ADR-0012](0012-observability.md), + [ADR-0014](0014-dashboard-auth-and-remediation.md), + [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md), + [ADR-0024](0024-project-intelligence-telemetry.md) + +## Context + +The dashboard currently answers three families of questions, each owned by its own context: + +- **Overview** — is the kit healthy and configured (subsystem status, hosts, routing, providers, + Intelligence's learning trends per [ADR-0024](0024-project-intelligence-telemetry.md))? +- **Usage** — what did sessions cost in tokens and API-equivalent dollars + ([ADR-0009](0009-usage-scorecard-local-transcript-analytics.md))? +- **Observability** — what are agents doing right now, and what did completed sessions do + ([ADR-0012](0012-observability.md))? + +Nobody answers the fourth family: **what does this toolchain cost the machine itself.** Real +questions a user of this stack has today, none answerable without hand-rolled `du`/`ps`/`find` +archaeology: + +- How big is the whole install — ruflo + agentic-qe + Claude Code + Codex + OpenCode + their + native addons and caches — and which tool owns how much of it? +- How much disk do six months of transcripts hold, broken down by host, by project, by session? + Which single sessions are the giants? +- How much of the disk is host ledgers and logs (Codex's `state_N.sqlite` thread ledgers, the + statusline tee files, OpenCode's store) as opposed to transcripts proper? +- How much RAM and CPU are three concurrent host processes plus background daemons consuming + right now? +- How many distinct skills, agents, and slash commands are actually deployed across hosts — and + which hosts have which? +- How many projects has this machine touched, and per project: how many lines of code, how much + disk (working tree, `.git`, `node_modules`)? + +Everything needed is already locally readable at trust boundaries the kit already crosses: + +- the install trees it manages and their install methods + ([MANAGED-TOOLS.md](../MANAGED-TOOLS.md); npm/mise/brew awareness already exists for drift + reporting); +- the current-user, argv-minimized process survey Observability already runs + (`src/lib/live/process-sessions.mjs`, hardened under + [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md)); +- the transcript roots Historical usage and Observability already walk + (`~/.claude/projects`, `~/.codex/sessions`, OpenCode's store); +- the catalog surfaces Integration management already projects into (`.claude/agents`, + `.claude/commands`, skills listings, OpenCode's converted agents, MCP registrations); +- the project catalog machine-wide discovery already assembles + ([ADR-0024](0024-project-intelligence-telemetry.md)'s transcript-cwd source). + +The gap is a **domain and a UX**, not access. + +## Decision + +### 1. A new bounded context: Machine footprint + +Machine footprint is its own bounded context — see +[Machine footprint](../ddd/machine-footprint.md) for the model, boundaries, and invariants. It is +deliberately **not** part of: + +- **Historical usage** — no tokens, no cost, no model identity here; a byte of transcript on disk + is a storage fact, not a spend fact; +- **Observability** — no session lifecycle, no evidence confidence, no transcript content; this + domain reads `stat` metadata and manifest names, never message bodies; +- **Project intelligence** — no learning counters; the shared piece is only project *discovery*, + reused as a candidate-path source exactly as ADR-0024 reuses Observability's workspace store. + +### 2. A fourth primary area: System + +The dashboard gains a fourth primary area — **System** — alongside Overview, Usage, and +Observability. This deliberately amends the "three stable primary areas" layout decision +([ADR-0005](0005-dashboard-in-page-routing-reveal.md) and `page.mjs`'s own header comment). +Machine-scoped resource facts are a peer question family, not a subsection of configuration +health; folding them under Overview would bury a whole domain under a tab whose contract is +"readiness and attention," and Overview's secondary rail is already five views deep. + +Proposed information architecture: + +```text +[ Overview | Usage | Observability | System ] + +System secondary rail: + [ Summary | Storage | Runtime | Catalog | Projects ] [ as of 2h ago · ⟳ rescan ] + +Summary KPI band: install size · data size · live processes (combined RSS) · projects · + skills/agents/commands counts · machine free-space denominator. + "Largest consumers" strip (top-N across all categories). Freshness label. +Storage Breakdown tree: category → host → project → session. Transcripts vs ledgers/logs vs + learning stores vs kit caches. Trailing-30d growth sparkline per host. Top-N largest + sessions/files. Advisory reclaimable candidates. +Runtime Live process table (host, pid, CPU%, RSS, uptime, bound project) + combined totals. + Daemon census (count, age vs TTL, budget state). Child/MCP server process count. +Catalog Deduplicated skills / agents / commands / plugins / MCP servers, each with a per-host + presence matrix (which hosts carry it). +Projects Table: project (name links to its git remote's web page when one exists — derived + from .git/config, "local only" otherwise), lines of code (by language), working-tree + bytes, .git bytes, node_modules bytes, last activity. +``` + +The window/refresh control sits in the secondary-actions slot, the same pattern as Usage's +day-window chips and Observability's history-window chips. + +### 3. Tiered, honest collection + +Two tiers, because the metrics differ by orders of magnitude in cost: + +- **Cheap tier** (served on every `GET /api/system`, TTL-cached ~60s like the project-snapshot + cache): the process census, sizes of individually known files (ledgers, caches, index files), + install-tree sizes carried forward from the last deep scan, and the persisted deep snapshot's + contents with their `asOf`. +- **Deep tier** (explicit, user-triggered): the full storage walk, per-project LOC counting, and + cross-host catalog deduplication. Single-flight — concurrent requests attach to the in-flight + scan. The result is persisted to `~/.config/agentic-kit/footprint-snapshot.json` so the panel + paints instantly on every later open, labeled with when it was measured. + +The honest-measurement contract of +[ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md) applies verbatim: a section +that has never been deep-scanned renders "not measured yet," never `0`; a measured zero is a real +zero; a subtree that fails to stat degrades that node to unknown without discarding its siblings. + +### 4. Initial metric taxonomy (non-exhaustive by design) + +Metrics marked ✚ are additions beyond the requesting examples; the taxonomy is expected to grow. + +| Section | Metric | Source | +|---------|--------|--------| +| Install | Per managed tool (ruflo, agentic-qe, claude, codex, opencode, ak, brain KB): version, install method, root path, tree bytes | managed-tools detection + walk | +| Install | Native-addon inventory and duplicate builds across trees (better-sqlite3, hnswlib, onnxruntime) ✚ | walk | +| Install | Shared caches: npx cache envs, brain KB, browser binaries ✚ | known roots | +| Install | Total install bytes + machine free-space denominator ✚ | walk + `statfs` | +| Runtime | Per live host process: pid, host, CPU%, RSS, uptime, bound project | existing runtime survey + `ps -o pcpu,rss` | +| Runtime | Daemon census: count, age vs 12h TTL, budget state ✚ | existing daemon registry | +| Runtime | Child / MCP-server process count ✚ | survey process tree | +| Storage | Transcript bytes + file counts: host → project → session | transcript-root walk | +| Storage | Host ledgers/logs: Codex `state_N.sqlite`, statusline tee, runtime-debug log, OpenCode store | known paths | +| Storage | Learning/memory stores: per-project `.claude-flow`, `.agentic-qe`, agentdb/HNSW/RVF files ✚ | project catalog + walk | +| Storage | ak's own caches: usage-index.json, observability-workspaces.json, footprint snapshot itself ✚ | known paths | +| Storage | Top-N largest sessions / files ✚ | walk | +| Storage | Trailing-30d growth per host (from mtime + size) ✚ | walk metadata | +| Storage | Advisory reclaimable candidates (stale npx envs, old transcripts, orphaned worktrees) ✚ | walk + heuristics | +| Catalog | Unique skills / agents / commands across hosts, per-host presence matrix | host catalog surfaces | +| Catalog | Plugins and registered MCP servers ✚ | settings surfaces | +| Catalog | Config surface: managed CLAUDE.md/AGENTS.md block count, settings file sizes ✚ | managed-blocks registry | +| Projects | Project count; per project: LOC by language, working-tree bytes, `.git` bytes ✚, `node_modules` bytes ✚, last activity | project catalog + walk | +| Projects | Git remote web link per project (origin URL → GitHub/GitLab/etc. page; "local only" when absent) ✚ | `.git/config` remote parse — the admin collector's `parseRepoSlug` shapes, reused | + +LOC counting is a zero-dependency extension-bucketed line count by the kit's own bounded walker +(no `cloc`/`tokei` dependency), excluding `node_modules`, vendored trees, and binary extensions — +stated as approximate, which is what the question needs. + +### 5. Delivery + +- `GET /api/system` — cheap tier + last persisted deep snapshot; same loopback bind, token auth + ([ADR-0014](0014-dashboard-auth-and-remediation.md)), and zero egress + ([ADR-0007](0007-maintainer-admin-local-telemetry.md)'s offline side of the line) as every + other dashboard route. +- `GET /api/system?refresh=deep` — starts or attaches to the single-flight deep scan. The + dashboard server is deliberately GET-only; a refresh is a re-*measurement* of local state, not + a mutation of user data, so it stays within that contract. +- `ak footprint [--deep] [--json]` — CLI parity sharing the same collector, following the + usage-scorecard precedent of one collector behind both surfaces. + +### 6. Read-only; reclaimables are advisory + +v1 computes reclaimable-space *candidates* and renders them with their rationale; it deletes +nothing. Cleanup remains CLI-owned where it already lives (`ak x daemon-gc`, npx cache tooling). +A future `ak footprint clean` would be its own decision with its own safety contract. + +### 7. A deliberate path-visibility exception + +Observability's `publicLivePayload` reduces absolute paths to leaf names because its payloads +describe *sessions* and a path is incidental provenance. In this domain the paths **are the +subject matter** — a storage breakdown that hides where the bytes live answers nothing. The +System payload therefore carries absolute paths, protected by the same token-gated loopback +delivery as everything else. File *contents* are never read: `stat` metadata, directory names, +catalog manifest names, and one narrow config read — `.git/config`'s remote URL, so a project +can link to its hosted repository page. Rendering that link stays inside the zero-egress +contract: the kit never fetches it; navigation is the user clicking a link in their own browser. + +## Consequences + +### Positive + +- The "what is this costing my machine" question family gets a real answer, from data the kit + already has trust-boundary access to — no new privileges, no egress. +- Sprawl becomes visible and actionable: duplicate native builds, giant sessions, stale caches, + and per-project bloat all surface with their locations. +- The four-family dashboard (health / spend / activity / footprint) completes a coherent mental + model, each family in its own context with clean boundaries. + +### Negative + +- A fourth primary area is a permanent navigation-surface expansion and an explicit amendment of + ADR-0005's three-area layout. +- Deep scans on large corpora (multi-GB transcript trees, many projects) take real time and I/O; + the tiered model contains but does not eliminate that cost. +- A persisted snapshot is a new on-disk artifact with its own staleness to manage honestly. + +### Risks and mitigations + +| Risk | Mitigation | +|------|------------| +| Deep scan runs long on huge corpora and reads as a hang | Single-flight with progress state surfaced in the panel; bounded walkers with entry caps; persisted snapshot means the panel is never blank while a scan runs | +| Stale snapshot presented as current | Every deep-tier figure carries `asOf`; the freshness label and rescan control are part of the Summary view's contract, not an afterthought | +| LOC figures treated as precise | Labeled approximate, extension-bucketed; the DDD doc forbids presenting them as authoritative | +| Walker follows a symlink cycle or escapes a root | Symlinks never followed; depth and entry caps; one bad subtree degrades to unknown for that node only | +| Scope creep into cleanup/mutation | v1 invariant: this context mutates nothing; reclaimables are advisory rows with rationale | +| Boundary erosion into Usage/Observability | DDD invariants forbid tokens/cost, session evidence, and content reads; cross-links replace duplication | + +## Open points for review + +1. **Naming** — primary tab **System** with CLI `ak footprint` is proposed; the alternative is + aligning both on one word (`ak system`, or a **Footprint** tab). +2. **Projects view** — kept separate from Storage because LOC is not a storage fact; could merge + if five sub-views feels heavy. +3. **Deep-scan freshness policy** — manual-only rescan is proposed; an alternative is + auto-refresh on dashboard open when the snapshot is older than N days. +4. **Windows** — the runtime census inherits the existing survey's unsupported-on-win32 honesty; + all other sections work everywhere. Acceptable for v1? + +## Follow-ups on acceptance + +- Add the context to [context map](../ddd/context-map.md) (new context row + Dashboard delivery + relationship) and merge the new terms into + [ubiquitous language](../ddd/ubiquitous-language.md). +- Add this ADR to the [ADR index](README.md) and the theme narrative. + +## References + +- [Machine footprint domain](../ddd/machine-footprint.md) (drafted alongside this ADR) +- [Design mock-up](../assets/system-tab-mock.html) — a self-contained, both-theme HTML mock of + the System area with illustrative data: per-metric chart forms (odometer KPIs, radial disk + gauge, donut, stacked bars, small-multiple growth areas, radar, presence matrix, composed + project bars), each card annotated with its form choice and rationale +- [Dashboard guide](../DASHBOARD.md) +- [Managed tools](../MANAGED-TOOLS.md) +- `src/lib/live/process-sessions.mjs` (the runtime survey this reuses) +- `src/lib/dashboard/project-discovery.mjs` (the project catalog this reuses) diff --git a/docs/assets/system-tab-mock.html b/docs/assets/system-tab-mock.html new file mode 100644 index 0000000..0aad1a3 --- /dev/null +++ b/docs/assets/system-tab-mock.html @@ -0,0 +1,643 @@ + + + + + +ak · System — design mock (ADR-0025) + + + + +
+ Design mock · illustrative data throughout — nothing here is measured + accompanies ADR-0025 + System primary area +
+ +
+ +

agentic-kit

v4.0.0-alpha.40 · local diagnostic panel
+
+ + + +
+
+ + + + + +
+
+ deep scan · 2h ago + +
+
+ +
+ + +
+
+ System +

Summary

+

The four families in one glance: install size, retained data, live resource use, and deployed + inventory — every deep-tier figure stamped with when it was measured.

+
+ +
+
+
Install footprint +
GB
+
7 managed tools · 318 native addons
+
Data retained +
GB
+
transcripts, ledgers, stores, caches
+
Live processes +
+
1.9 GB RSS · 11% CPU combined
+
Projects +
+
412k LOC across catalog
+
Catalog +
+
skills · 23 agents · 41 commands
+
+ +
+

Disk denominator

radial gauge
+ + + + + + 12.7 GB + toolchain · of 994 GB disk + 213 GB free + + +
Why a gauge: one ratio against a limit. The denominator keeps "3.8 GB + install" honest — big enough to see, small enough to not panic about. Accent = toolchain; + everything else stays de-emphasis gray so the accent has one meaning.
+
+ +
+

Largest consumers — all categories

ranked bars
+
+
codex transcripts
3.9 GB
+
claude transcripts
2.5 GB
+
ruflo install tree
1.4 GB
+
codex state ledgers
1.1 GB
+
learning stores (all projects)
0.9 GB
+
+
Why ranked bars: "what's eating the disk" is a magnitude comparison — + the top-5 across every category beats five per-category charts for a summary. Host hue + carries identity where a row belongs to one host; install/shared rows stay neutral.
+
+
+
+ + +
+
+ System +

Storage

+

Retained data by category, then by host, down to the session leaf — plus growth, giants, and + advisory reclaimables. Paths are the answer here, so rows carry them.

+
+ +
+
+

Retained data by category

donut
+
+ + + + + + + + 8.9 + GB retained + +
+ transcripts 5.2 GB · 58% + ledgers & logs 2.0 GB · 22% + learning stores 1.0 GB · 12% + kit caches 0.7 GB · 8% +
+
+
Why a donut: part-to-whole with exactly four slices and a center total — + within honest-pie territory. Values ride the legend (light-mode aqua/yellow sit under 3:1, so + the relief rule makes labels mandatory, never color-alone). More than ~5 categories → switch + to a stacked bar.
+
+ +
+

Per-host split by category

stacked bars
+
+
codex +
+ + + +
5.2 GB
+
claude +
+ + + +
3.2 GB
+
opencode +
+ + +
0.5 GB
+
+
transcripts + ledgers & logs + learning stores + kit caches
+
Why horizontal stacks: the same four category hues as the donut — color + follows the entity across the whole view, so the reader learns the mapping once. Rows share + one scale; 2px gaps separate segments. Clicking a row would drill to project → session leaves.
+
+ +
+

Growth — bytes added per day, 30d

small-multiple areas
+
+
claude
+ + + + + + +
+
codex
+ + + + + + +
+
opencode
+ + + + + + +
+
+
Why small multiples, not one chart: three hosts on one axis would make + opencode invisible against codex. Each panel keeps its own emphasized endpoint; host hue + matches the host's color everywhere else. Derived from mtime + size only — no content reads.
+
+ +
+

Reclaimable — advisory only

annotated list
+
+
⚠ 1.9 GB +
Transcripts older than 180d (312 files)
~/.codex/sessions/2026-0[1-2]/**
+
⚠ 640 MB +
Stale npx cache envs (9, unused > 90d)
~/.npm/_npx/*
+
⚠ 210 MB +
Orphaned worktrees (2, branch merged)
~/dev/keel/.autopilot/wt-*
+
+
Why a list, not a chart: each row is a decision, not a magnitude — the + rationale and the path are the content. Warning color ships with icon + label, never alone. + No delete button exists by invariant; the row points at the CLI that owns cleanup.
+
+ +
+

Largest sessions

table + inline bars
+
+ + + + + + + +
SessionHostProjectFilesSizeShare of host
rollout-2026-08-01T09-14…codexagentic-kit61412 MB
94d88189-5c24-420a…claudeagentic-kit18287 MB
rollout-2026-07-28T14-02…codexemailibrium44231 MB
a5e3220a-2e76-4e94…claudekeel9164 MB
+
Why a table: a session id, host, project, and path are lookup facts — the + bar is a garnish for scanning, the row is the unit. Doubles as the accessibility table view for + this section's charts.
+
+
+
+ + +
+
+ System +

Runtime

+

A point-in-time census — computed on request, never persisted. CPU and memory of live host + processes, daemons against their TTL, and combined totals.

+
+ +
+
+

Live host processes

table + inline bars
+
+ + + + + + + + + + +
HostpidProjectUptimeCPURSS
claude48112agentic-kit3h 12m6.2%
1.31 GB
codex50871emailibrium1h 40m3.9%
0.48 GB
opencode51230keel22m0.8%
0.14 GB
+
Why a table with bars, not gauges per process: the census is a scan-and- + compare surface — bars share one RSS scale so "claude is 3× codex" is visible without reading + numbers. Reuses the argv-minimized survey; a row is a resource consumer, not a session actor.
+
+ +
+

Combined memory

meter
+
+
1.93GB RSS combined
+
+
of 32 GB physical · 11.9% CPU combined (8 cores)
+
+

Daemons

stat tiles
+
+
+
2
+
running · oldest 4.2h of 12h TTL
+
+
$0
+
AI workers off · budget idle
+
+
Why a meter, not a radial: one ratio against a hard limit (physical RAM) + reads fastest as a straight track. The daemon tiles are single current values — stat tiles, not + a one-bar chart.
+
+
+
+ + +
+
+ System +

Catalog

+

What is actually deployed, deduplicated across hosts — and which host carries what.

+
+ +
+
+

Host inventory profile

radar · axis-normalized
+ + + + + + + + + + + + + + + skills + agents + commands + plugins + MCP + + +
claude + codex + opencode
+
Why a radar (and its caveat): the question is shape — "is codex a + commands-heavy host?" — not precise magnitude. Three series is the all-pairs-safe cap for + overlapping fills. Each axis is normalized to its own max, and the tooltip carries the real + counts, because radar area is not a quantity.
+
+ +
+

Unique across hosts

stat row + presence matrix
+
+
+
57
unique skills
+
+
23
unique agents
+
+
41
unique commands
+
+
12
plugins
+
+
9
MCP servers
+
+
+ claudecodexopencode + security-testing + qe-court + spring-m11n:migrate + autopilot:orchestrate + frontend-design +
+
…52 more · search + full table in the real view
+
Why a matrix, not more charts: "which hosts carry it" is boolean identity — + a dot grid answers per-item questions the radar's aggregate cannot. Counts get stat tiles + because each is a single current value. Observed inventory only; never desired state.
+
+
+
+ + +
+
+ System +

Projects

+

Every project the machine knows, sized two ways — code and disk — with the overhead + (.git, node_modules) kept separate so it can't + masquerade as "your project got big."

+
+ +
+
+

Project footprints

table + composed bars
+
+ LOC: JS/MJS + TS + Rust + other + Disk: tree + .git + node_modules +
+
+ + + + + + + + + + + + + + + + + + + +
ProjectLOC ≈By languageDisktree · .git · node_modulesLast active
agentic-kit ↗
github · pacphi/agentic-kit
96k
1.9 GB
2h ago
emailibrium ↗
github · pacphi/emailibrium
201k
3.4 GB
1d ago
keel
local only — no git remote
64k
1.1 GB
3d ago
+
…11 more projects · sortable by any column in the real view
+
Two bars, two color jobs: LOC is identity (languages → categorical + slots, "other" folds to gray past three). Disk composition is one entity's ranked parts — + shades of a single hue, darkest = the part you wrote, faintest = reinstallable overhead. LOC is + labeled approximate by invariant; extension-bucketed, stated exclusions. + Remote links: the project name is the link (one obvious click target, ↗ marks it as + leaving the panel), the host · slug subline carries provenance, and + the tooltip holds the raw origin URL — parsed from .git/config via + the same shapes parseRepoSlug already handles. No remote renders an + explicit "local only", never a dead link; the kit itself never fetches the URL.
+
+
+
+ +
ak · System — design mock for ADR-0025 · illustrative data · both themes via the dashboard's own tokens
+
+ + + + + + diff --git a/docs/ddd/machine-footprint.md b/docs/ddd/machine-footprint.md new file mode 100644 index 0000000..dad9e00 --- /dev/null +++ b/docs/ddd/machine-footprint.md @@ -0,0 +1,236 @@ +# Machine Footprint Domain + +> **Draft for review.** This document specifies the domain proposed by +> [ADR-0025](../adr/0025-machine-footprint-metrics.md). Nothing described here is implemented +> yet; module paths named below are the intended homes, not existing files. On acceptance, the +> new terms merge into [Ubiquitous language](ubiquitous-language.md) and the context joins the +> [context map](context-map.md). + +## Purpose + +Machine footprint answers **what this toolchain costs the machine itself**: how many bytes the +managed install occupies and where; how much CPU and RAM live host processes and daemons are +consuming right now; how retained data (transcripts, ledgers, logs, learning stores, caches) +breaks down by category, host, project, and session; and what is actually deployed — the +deduplicated inventory of skills, agents, commands, plugins, and MCP servers across hosts, plus +every known project's size in lines of code and disk. + +It is a read-only measurement domain over local state the kit already has trust-boundary access +to. It renders in the dashboard's proposed **System** primary area and through a CLI twin +(`ak footprint`), and it mutates nothing — including the reclaimable-space candidates it +computes, which are advisory rows with rationale, never delete actions. + +The shared terms in [Ubiquitous language](ubiquitous-language.md) are normative; the +[terms table](#ubiquitous-language-additions) below is this draft's proposed addition to it. + +## Why this is a separate context + +Each neighboring context owns a different kind of fact about overlapping raw material, and the +boundaries are what keep all four honest: + +- **[Historical usage](context-map.md)** owns *spend* facts — tokens, API-equivalent cost, model + identity — parsed from transcript **content**. Machine footprint reads transcript **metadata** + (`stat` size, mtime, path) and never opens a message body. A 400 MB session is a storage fact + here and a token fact there; the two figures answer different questions and neither substitutes + for the other. +- **[Observability](observability.md)** owns *activity* evidence — session lifecycle, actors, + per-field confidence, a protected transcript plane. Machine footprint's runtime census reuses + the same process survey as a *source*, but publishes resource rows (CPU%, RSS, uptime), not + `ObservedSession` evidence; a process here is a consumer of memory, not an actor in a session + graph. +- **[Project intelligence](project-intelligence.md)** owns *learning* trends from + `.claude-flow/` state. Machine footprint measures those same directories only as bytes on + disk. The shared piece is project *discovery* — the candidate-path catalog — reused exactly as + ADR-0024 reuses Observability's workspace store: a path is not evidence, and discovery supplies + nothing this domain renders as a measurement. +- **[Integration management](integration-management.md)** owns what *should* be deployed + (bindings, projections, ownership). Machine footprint reports what *is* on disk and how big it + is; catalog counts here are observed inventory, never desired state. + +Because every source is the local filesystem and the current-user process table — the same trust +boundary `ak status` and the runtime survey already cross — no anti-corruption adapter guards +this domain's reads. What *is* guarded is content: this domain's collectors are structurally +metadata-only (they read directory entries, `stat` results, and manifest *names*), so transcript +text, prompt content, and tool payloads cannot enter the model at all. + +## Model + +```text +Sources (all local, all metadata-only) + install roots (managed-tools detection) process table (runtime survey + pcpu/rss) + transcript roots (~/.claude, ~/.codex, opencode store) + known files (ledgers, tee logs, caches, indexes) + project catalog (discovery reuse) host catalog surfaces (agents/skills/commands/MCP) + | + v +Collectors (bounded walkers; two tiers) + cheap tier: process census + known-file stats + carry-forward of last deep scan (TTL ~60s) + deep tier: full storage walk + LOC count + catalog dedup (explicit, + single-flight) + | + v +FootprintSnapshot { asOf, completeness, install, runtime, storage, catalog, projects } + install: HostInstallation[] { tool, version, installMethod, root, bytes, nativeAddons[] } + runtime: RuntimeCensus { processes[], daemons[], totals } (ephemeral, never + persisted) + storage: StorageBreakdown { nodes: category → host → project → session, growth, topN, + reclaimables[] } + catalog: CatalogInventory { skills[], agents[], commands[], plugins[], mcpServers[], + each with per-host presence } + projects: ProjectFootprint[] { path, label, remote?: {host, slug, webUrl}, loc: {language: + lines}, treeBytes, gitBytes, nodeModulesBytes, lastActivity } + | + v +Delivery + GET /api/system → cheap tier + persisted snapshot (token auth, loopback, no egress) + GET /api/system?refresh=deep → start-or-attach the single-flight deep scan + ak footprint [--deep] [--json] → the same collector, CLI-rendered + | + v + System primary area: Summary | Storage | Runtime | Catalog | Projects +``` + +### Measurement semantics + +Every figure in this domain is a `Measurement` — a value plus how it was obtained: + +- **measured** — a real walk/stat/count produced it, stamped with the snapshot's `asOf`; +- **carried forward** — from the persisted snapshot, presented with that snapshot's `asOf`, + never as current; +- **unknown** — never measured, or the measurement failed for that node; rendered as + "not measured yet" / "unavailable," **never as `0`**. + +A measured zero is a real zero and renders as one. This is the same zero-vs-unknown discipline +the dashboard's `metric()` helper and [ADR-0023](../adr/0023-fail-closed-operations-and-explicit-degradation.md) +already enforce elsewhere. + +### Install footprint + +One `HostInstallation` per managed tool (ruflo, agentic-qe, Claude Code, Codex, OpenCode, +agentic-kit itself, the brain KB), carrying the version and install method the kit's existing +detection already knows (npm/mise/brew/self-managed), the resolved root path, tree bytes from +the deep walk, and a native-addon inventory — including the *duplicate-builds* view (the same +native module compiled into multiple trees), which is sprawl the user cannot see today. Shared +caches (npx envs, browser binaries) are install-adjacent nodes with their own rows, not smeared +into a tool's tree. The machine's free-space figure is the section's denominator, so "the +install is X GB" always has a "of Y free" next to it. + +### Runtime census + +A point-in-time table of live host processes — reusing the existing current-user, argv-minimized +survey and extending its `ps` read with `pcpu`/`rss` — plus the daemon census (count, age +against the 12h TTL, budget state) and a child/MCP-server process count. The census is +**ephemeral**: computed per request, never persisted into the snapshot file, because a process +table is a moment, not a fact worth retaining, and persisting it would create a stale-liveness +trap. On Windows the census degrades to an honest "unsupported," matching the survey it reuses; +every other section still works. + +### Storage breakdown + +A tree of `StorageNode`s: category (transcripts / ledgers-and-logs / learning stores / kit +caches) → host → project → session leaf, each with bytes and file count. Derived views over the +same walk: trailing-30d growth per host (from mtime + size — no content reads), top-N largest +sessions and files, and advisory `ReclaimableCandidate` rows (stale npx envs, transcripts beyond +a stated age, orphaned worktrees), each carrying its rationale and its path. Candidates are +information, not actions — this context has no delete verb. + +### Catalog inventory + +Deduplicated `CatalogItem`s across hosts — skills, agents, commands, plugins, MCP servers — +keyed by normalized name, each with a per-host presence matrix (which hosts carry it, from which +surface it was observed). Counting is by manifest/directory-entry **names** on the host catalog +surfaces Integration management already projects into; item file contents are not parsed beyond +what naming requires. The config-surface row (managed CLAUDE.md/AGENTS.md block count, settings +file sizes) lives here because it answers the same "what is deployed" question. + +### Project footprint + +One `ProjectFootprint` per project in the shared discovery catalog: approximate lines of code +bucketed by language (the kit's own extension-bucketed walker — explicitly approximate, excluded +trees stated: `node_modules`, vendored, binary extensions), working-tree bytes, `.git` bytes, +`node_modules` bytes (kept separate precisely because it dominates and distorts), and last +activity. LOC figures are labeled approximate wherever they render; this domain forbids +presenting them as authoritative. + +A project additionally carries an optional `remote` — host, slug, and derived web URL — parsed +from `.git/config`'s origin remote through the same URL-shape handling the admin collector's +`parseRepoSlug` already proves (git+https / ssh / scp / bare). In the Projects table the project +name renders as a link to that page, with the raw remote in the tooltip; a recognized host +(GitHub, GitLab, Bitbucket, or a self-hosted URL that is already web-shaped) yields a link, an +unrecognized remote shape renders the remote name unlinked, and a project with no remote renders +an explicit "local only" — absence stated, never guessed. The link is user-initiated browser +navigation; the kit itself never fetches the remote. + +## Delivery + +`GET /api/system` follows Dashboard delivery's existing contract: loopback bind, per-session +token auth ([ADR-0014](../adr/0014-dashboard-auth-and-remediation.md)), `no-store`, zero egress. +The response is the cheap tier computed fresh (TTL ~60s, shared-cache pattern like the +project-snapshot cache) merged with the persisted deep snapshot and its `asOf`. +`?refresh=deep` starts the deep scan or attaches to the one in flight (single-flight, like the +usage index's coalesced builds); progress is surfaced so a long scan reads as working, not hung. +The server stays GET-only: a rescan re-measures local state and writes only this domain's own +snapshot file — it mutates no user data. + +One deliberate divergence from Observability's delivery: absolute paths are **part of this +payload**. `publicLivePayload`'s leaf-only rule exists to keep incidental provenance out of +session payloads; here the path is the answer ("where are the bytes"), and the same token-gated +loopback delivery protects it. Transcript/message content remains structurally absent — the +collectors never read it, so delivery cannot leak it. + +The CLI twin (`ak footprint`) renders the same collector output, `--json` emitting the snapshot +shape verbatim, following the one-collector-two-surfaces precedent of the usage scorecard. + +## Invariants + +1. **Metadata only, ever.** Collectors read directory entries, `stat` results, manifest names, + and `.git/config`'s remote URL. No transcript, prompt, message, or tool-payload content + enters this domain, in any tier, on any path. +2. **Unknown is never zero.** An unmeasured or failed measurement renders as unknown with a + reason; a measured zero renders as zero. No fabricated figures. +3. **Freshness is part of the value.** Every deep-tier figure carries the snapshot `asOf` it came + from; carried-forward data is never presented as current. +4. **This context mutates nothing.** No delete, prune, or cleanup verb exists here; reclaimable + candidates are advisory rows with rationale. (The snapshot file it owns is the sole write.) +5. **The runtime census is ephemeral.** It is computed per request and never persisted; a stale + process table is never replayed as liveness. +6. **Bounded walkers.** Symlinks are never followed; depth and entry caps apply; one unreadable + subtree degrades that node to unknown without discarding siblings or aborting the scan. +7. **Single-flight deep scans.** Concurrent refresh requests attach to the in-flight scan; two + scans never race each other or double-write the snapshot. +8. **No spend, no activity, no learning facts.** Tokens/cost stay in Historical usage; session + lifecycle/evidence stays in Observability; learning counters stay in Project intelligence. + Cross-links, not duplication. +9. **Discovery supplies paths, not measurements.** The shared project catalog contributes + candidate locations only; everything rendered is measured by this domain's own collectors. +10. **Catalog counts are observed inventory.** They state what is on disk per host surface, + never desired state, and never upgrade Integration management's ownership facts. +11. **LOC is approximate and says so.** Extension-bucketed line counts with stated exclusions; + no rendering presents them as authoritative. +12. **Same delivery protections as the rest of the dashboard.** Loopback, token auth, GET-only, + zero egress; the absolute-path exception is deliberate, documented, and content-free. + +## Ubiquitous language additions + +| Term | Meaning | +|------|---------| +| Footprint | The machine-resource cost of the toolchain: install bytes, runtime CPU/RSS, retained-data bytes, deployed inventory | +| FootprintSnapshot | The persisted result of a deep scan: `asOf`, completeness, and the four section models | +| Measurement | A value plus provenance: measured (with `asOf`), carried forward, or unknown-with-reason — unknown is never zero | +| HostInstallation | One managed tool's install facts: version, install method, root, tree bytes, native addons | +| RuntimeCensus | The ephemeral point-in-time table of live host processes, daemons, and totals | +| StorageNode | One node in the category → host → project → session breakdown: bytes + file count | +| ReclaimableCandidate | An advisory row naming reclaimable space, its path, and its rationale — never an action | +| CatalogItem | A deduplicated deployed artifact (skill, agent, command, plugin, MCP server) with a per-host presence matrix | +| ProjectFootprint | One project's size facts: approximate LOC by language, tree/`.git`/`node_modules` bytes, last activity, and an optional git-remote web link ("local only" when absent) | +| Deep scan | The explicit, single-flight full measurement pass that produces a FootprintSnapshot | +| Cheap tier | The per-request census + known-file stats + snapshot carry-forward served on every read | + +## References + +- [ADR-0025](../adr/0025-machine-footprint-metrics.md) — the decision record this draft + accompanies +- [Context map](context-map.md) — joined on acceptance +- [Historical usage / Observability / Project intelligence](context-map.md) — the neighboring + contexts this domain is deliberately distinct from +- [Dashboard guide](../DASHBOARD.md) From ed6047fbb3b7e8c0a72979ea203b11ff64febe8b Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 13:28:21 -0700 Subject: [PATCH 04/19] =?UTF-8?q?docs:=20propose=20the=20About=20area=20?= =?UTF-8?q?=E2=80=94=20ADR-0026,=20component-directory=20domain,=20design?= =?UTF-8?q?=20mock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drafts for review; nothing implemented yet. - ADR-0026 (Proposed): a leftmost About primary area introducing every component ak installs or configures — curated editorial copy joined with existing detection facts (no new endpoint, no probing), a registry↔directory parity gate so a managed tool cannot ship without its About card, official host marks + honest monogram tiles, and outbound user-initiated links inside the zero-egress contract - docs/ddd/component-directory.md: the editorial/detection split as the load-bearing boundary, the new-user register contract (~50-word plain-language paragraphs, no runtime claims in prose), 10 invariants, and proposed ubiquitous-language terms - docs/assets/about-tab-mock.html: self-contained both-theme mock on the dashboard's own tokens — hero orientation strip with a how-it-fits map, category card grid (hosts first, honest not-installed state shown, configured surfaces with manage: commands), per-section design notes, and an annotated card anatomy --- docs/adr/0026-about-component-directory.md | 159 +++++++ docs/assets/about-tab-mock.html | 469 +++++++++++++++++++++ docs/ddd/component-directory.md | 169 ++++++++ 3 files changed, 797 insertions(+) create mode 100644 docs/adr/0026-about-component-directory.md create mode 100644 docs/assets/about-tab-mock.html create mode 100644 docs/ddd/component-directory.md diff --git a/docs/adr/0026-about-component-directory.md b/docs/adr/0026-about-component-directory.md new file mode 100644 index 0000000..8eb9a04 --- /dev/null +++ b/docs/adr/0026-about-component-directory.md @@ -0,0 +1,159 @@ +# ADR-0026 — About: a component directory that explains everything ak installs + +- **Status:** Proposed (draft for review — no implementation exists yet) +- **Date:** 2026-08-06 +- **Deciders:** agentic-kit maintainers +- **Related:** [ADR-0005](0005-dashboard-in-page-routing-reveal.md), + [ADR-0007](0007-maintainer-admin-local-telemetry.md), + [ADR-0016](0016-capability-driven-integration-adapters.md), + [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md), + [ADR-0025](0025-machine-footprint-metrics.md) + +## Context + +agentic-kit installs and configures a lot on a user's behalf: three frontier-agent CLIs, an +orchestration engine, a memory layer, a quality fleet, security scanners, an offline knowledge +base, MCP registrations, guidance blocks, statuslines, a routing policy, background daemons. +Every existing dashboard area assumes the user already knows what these things *are*: + +- **Overview** grades their health, **Usage** their spend, **Observability** their activity, and + the proposed **System** ([ADR-0025](0025-machine-footprint-metrics.md)) their machine cost — + all operational views of components the user is presumed to recognize. +- The only place that *introduces* the components is prose documentation + ([MANAGED-TOOLS.md](../MANAGED-TOOLS.md), [SETUP.md](../SETUP.md)) — maintainer-register + reference material, not a new user's first five minutes. + +A new user's first honest question is prior to all of that: **"what did this thing just put on +my machine, and why should I be glad it's there?"** Nothing answers it today. The result is a +trust gap exactly where trust matters most — at first contact, right after `ak setup` printed a +long list of installs. + +## Decision + +### 1. A new leftmost primary area: About + +The dashboard gains a fifth primary area — **About** — placed **left of Overview**, first in +the tab order. This amends [ADR-0005](0005-dashboard-in-page-routing-reveal.md)'s layout a +second time (after ADR-0025's fourth area). Leftmost placement is deliberate: About is the +reading-order entry point for someone who doesn't yet know what the other tabs are about. +**Overview remains the default landing view** — About is discoverable first position, not a +gate returning users must click past. + +```text +[ About | Overview | Usage | Observability | System ] + +About (no secondary rail — one scrolling page with category anchors): + Hero One friendly sentence of orientation + component count + a five-word map of how + the pieces relate (hosts ⟷ engine ⟷ memory/quality/security, kit around all). + Hosts The agent CLIs the user already recognizes — Claude Code, Codex, OpenCode. + Engine & ruflo (orchestration, cross-session memory, learning), agentdb (the memory + memory layer, version-pinned to ruflo). + Quality agentic-qe — test generation, coverage, quality gates. + Safety @claude-flow/aidefence + @claude-flow/security — prompt-injection defense and + scanning. + Knowledge RuvNet Brain — the offline KB that grounds answers about this stack. + The kit agentic-kit itself — the caretaker that installs, heals, and explains the rest. + Configured for you Non-package surfaces: MCP registrations, guidance blocks, + statuslines, dual-host routing & bridge, background daemon, permission + allowlists — each with "configured by setup/sync · yours to change". +``` + +### 2. A new bounded context: Component directory + +Curated editorial content joined with live detection facts — see +[Component directory](../ddd/component-directory.md). The split is the decision: + +- **Editorial** (this context owns): per-component tagline, one friendly paragraph of value + proposition, outbound links (GitHub / npm / docs), icon spec, category, and curated order. + Checked into the repo as a directory module, versioned with the release — never generated at + runtime, never fetched. +- **Detection** (borrowed, read-only): installed/not-installed, version, install method — + the facts `/api/status` and the managed-tools machinery already produce. About performs + **no probing of its own and adds no endpoint**; the page renders the directory module joined + client-side with the existing status payload. + +### 3. Completeness is a gate, not a hope + +Every tool in the managed-tools registry must have a directory entry — enforced by a parity +test, so a newly managed tool cannot ship without its About card. The reverse also holds: a +directory entry for something ak does not install or configure is a lie and fails the same +gate. + +### 4. Card anatomy and the new-user register + +Each card: an icon tile; the component name with an honest state chip (`installed v3.34.0` · +`not installed — ak setup adds it` · `configured`); a bold plain-language tagline; **one** +paragraph (~50 words) of value proposition written to a reader who has never heard of the +tool; and a row of link pills (GitHub / npm / Docs). Editorial register is a contract, not a +style hope: friendly, concrete, jargon-free — every term of art either avoided or explained in +the sentence that uses it ("agent swarms" → "teams of specialist agents"). No marketing +superlatives; the value prop states what the thing *does for the user*. + +### 5. Iconography: official marks where they exist, honest monograms elsewhere + +The three hosts reuse the exact official SVG marks the dashboard already ships for +Observability's session rows. Components without an official mark get a **monogram tile** +(letterform on a category-hued rounded tile) — deliberately not a fabricated logo. All icons +are inline SVG/CSS (the dashboard is self-contained; no remote images), and the same icon for +a component everywhere it appears. + +### 6. Links are outbound and user-initiated; the kit stays offline + +Every link is a plain `https` anchor the user clicks in their own browser — GitHub repo, npm +package page, public docs. The kit fetches nothing ([ADR-0007](0007-maintainer-admin-local-telemetry.md)'s +offline side), link URLs live in the versioned directory module, and the nightly external link +check covers them like every other doc link. + +## Consequences + +### Positive + +- The first-five-minutes trust gap closes: every installed thing explains itself, in one + place, in plain language, with the receipts (source, package, docs) one click away. +- The parity gate turns "docs drift from reality" into a test failure instead of a review hope. +- Zero new collection surface: no endpoint, no probe, no cache — editorial content plus a join + with facts the dashboard already has. + +### Negative + +- A fifth primary tab; the tab bar is reaching its comfortable ceiling, and any sixth area + should trigger a navigation rethink rather than another amendment. +- Curated copy is a maintenance duty: component descriptions, links, and value props must be + kept truthful as upstreams evolve (the parity gate catches presence, not prose accuracy). +- Editorial tone is subjective; the register contract reduces but cannot eliminate review churn. + +### Risks and mitigations + +| Risk | Mitigation | +|------|------------| +| Copy claims something is installed when it isn't | Editorial/detection split is structural: state chips render only detection facts; copy is forbidden (by DDD invariant and review checklist) from asserting runtime state | +| A new managed tool ships without an About card | Registry↔directory parity test fails the build | +| Links rot | Nightly external link check already covers `docs/**`; the directory module's URLs join that sweep | +| Brand misrepresentation via invented logos | Official marks only where already shipped as official; everything else is an explicit monogram tile | +| The page drifts into a second status/metrics view | DDD invariant: no numbers beyond version strings — health belongs to Overview, cost to Usage, footprint to System | + +## Open points for review + +1. **First-run behavior** — Overview stays the default; should the very first dashboard open + (no prior localStorage) land on About once, or show a dismissible "new here? start with + About" nudge instead? +2. **CLI twin** — `ak about` printing the same directory (with state chips) is cheap and + symmetric with the usage/footprint precedent; worth having in v1? +3. **Configured-surfaces depth** — one card per surface as proposed, or one "Configured for + you" card with an expandable list? + +## Follow-ups on acceptance + +- Add the context to the [context map](../ddd/context-map.md) and merge terms into + [ubiquitous language](../ddd/ubiquitous-language.md); add this ADR to the + [index](README.md) and theme narrative. +- Wire the registry↔directory parity test alongside the managed-tools tests. + +## References + +- [Component directory domain](../ddd/component-directory.md) (drafted alongside this ADR) +- [Design mock-up](../assets/about-tab-mock.html) — self-contained both-theme HTML mock with + the full card grid, category sections, per-section design rationale, and an annotated card + anatomy +- [Managed tools](../MANAGED-TOOLS.md) — the registry this directory must stay in parity with +- [Dashboard guide](../DASHBOARD.md) diff --git a/docs/assets/about-tab-mock.html b/docs/assets/about-tab-mock.html new file mode 100644 index 0000000..b1007ee --- /dev/null +++ b/docs/assets/about-tab-mock.html @@ -0,0 +1,469 @@ + + + + + +ak · About — design mock (ADR-0026) + + + + +
+ Design mock · illustrative copy, versions, and links — final content lives in the versioned directory module + accompanies ADR-0026 + About primary area +
+ +
+ +

agentic-kit

v4.0.0-alpha.40 · local diagnostic panel
+
+ + + + + +
+ +
+ About +

Meet your toolkit

+

agentic-kit set up 9 components and 6 configurations on this machine. + This page says what each one is, in plain words — and where to read more. Health lives in + Overview, spend in Usage, activity in Observability; here, everything just introduces itself.

+
+ How it fits +
+
Coding agentsClaude Code · Codex · OpenCode
+ ⇆ +
Engine + memoryruflo · agentdb
+ ⇆ +
Quality · Safety · Knowledgeagentic-qe · aidefence · Brain
+ ⤺ +
agentic-kitinstalls & heals it all
+
+
+
+ + +
+ About +

Hosts — the agents you talk to

+

The coding agents themselves. Everything further down exists to make these + smarter, safer, and easier to watch.

+
+
+
+ + Claude Codeinstalled · v2.1.223 +
+ Anthropic's coding agent, in your terminal. +

Point it at a repository and talk — it reads your code, writes changes, runs + the tests, and explains what it did as it goes. Most of what agentic-kit sets up exists to + make this tool and its peers below work better together.

+
GitHub ↗ + npm ↗ + Docs ↗
+
+
+
+ + Codexinstalled · v0.29.0 +
+ OpenAI's coding agent — a second pair of eyes. +

The same idea from OpenAI: an agent that codes with you in the terminal. With + both installed, agentic-kit runs them as peers — each can lead, review the other's work, or + pick up when one gets stuck.

+
GitHub ↗ + npm ↗ + Docs ↗
+
+
+
+ + OpenCodenot installed — ak setup adds it +
+ An open-source coding agent, supervised. +

A community-built agent CLI that agentic-kit can manage as an opt-in worker: + explicitly routed for specific jobs, observable like the others, and never handed the lead + by accident.

+
GitHub ↗ + npm ↗ + Docs ↗
+
+
+
Design note: hosts lead because they're what a new user already + recognizes — familiar things first, infrastructure after. Official marks are reused + byte-identically from Observability's session rows. The OpenCode card deliberately shows the + honest not-installed state: absence is information, not an empty slot.
+
+ + +
+ About +

Engine & memory — what makes sessions smarter

+
+
+
+ rf + rufloinstalled · v3.34.0 +
+ The orchestration engine underneath. +

Gives your agents what a single session can't: memory that survives restarts, + teams of specialists working in parallel, hooks that learn your patterns, and security + scanning along the way. agentic-kit keeps it current and healthy.

+
GitHub ↗ + npm ↗
+
+
+
+ db + agentdbinstalled · 3.0.0-alpha.20 · pinned to ruflo +
+ Where agent memory actually lives. +

A fast, local database for what your agents learn and remember — no server, no + cloud. It's version-matched to ruflo on purpose, so the engine and its memory can never + drift apart.

+
GitHub ↗ + npm ↗
+
+
+
Design note: monogram tiles (letters on a category hue) are the honest + icon for tools without an official mark — consistent, self-contained under the CSP, and never a + fabricated logo. The pinned-version chip explains itself inline rather than hiding a deliberate + choice behind "outdated".
+
+ + +
+ About +

Quality · Safety · Knowledge — evidence, defense, grounding

+

Three specialists with one theme: making agent work trustworthy — tested, + protected from smuggled instructions, and answered from real source.

+
+
+
+ qe + agentic-qeinstalled · v3.13.9 +
+ A quality-engineering fleet on call. +

Specialist agents for the unglamorous parts: generating tests, finding + coverage gaps, and gating changes on real quality signals — so "the agent says it works" + turns into evidence you can check.

+
GitHub ↗ + npm ↗
+
+
+
+ + aidefence + securityinstalled · bundled with ruflo +
+ Checks text before your agents trust it. +

A pasted issue, a web page, a stranger's README — any of them can carry + instructions smuggled into content ("prompt injection"). These scanners check untrusted + text before it reaches your agents. ak reinstalls them if an upgrade ever drops them.

+
GitHub ↗ + npm ↗
+
+
+
+ B + RuvNet Braininstalled · v0.9.4 +
+ Answers about these tools, from their real source. +

The tools on this page evolve faster than any AI model's training data. The + Brain is a local, offline knowledge base built from their actual source code — so when you + ask your agent about this stack, it answers from evidence, not stale memory.

+
GitHub ↗
+
+
+
Design note: three one-tool categories share a row instead of three + sparse sections — the hero map already groups them as one cluster. The shield is the page's one + hand-drawn glyph ("safety" is where a monogram would undersell the job), and the aidefence + paragraph defines "prompt injection" in the same breath it uses it: the register contract in + action.
+
+ + +
+ About +

The kit — who takes care of all this

+
+
+
+ + agentic-kitinstalled · v4.0.0-alpha.40 +
+ The caretaker for everything above. +

One command installs the set; one command heals it after upgrades; this + dashboard shows what it's all doing. If a tool on this page breaks, drifts, or goes + missing, ak notices and says so — that is its whole job.

+
GitHub ↗ + npm ↗
+
+
+
+ + +
+ About +

Configured for you

+

Not packages — settings and wiring ak set up on your behalf. Each names the + command that manages it: yours to change, never a black box.

+
+
+
⎈ + MCP registrationsconfigured
+

Agents reach tools through MCP — the plug-in protocol for AI tooling. ak + registers this stack's tools once, at user scope, so every project gets them without + per-repo setup.

+
manage: ak x mcp pick
+
+
+
+ Guidance blocksconfigured
+

Short managed sections in CLAUDE.md and AGENTS.md that teach each agent the + house rules for this stack. ak keeps them current — and never touches text you wrote + yourself.

+
manage: ak sync
+
+
+
⏯ + Statuslinesconfigured
+

A live footer inside Claude Code and Codex showing the model in use, session + usage, and remaining limits at a glance — so you never wonder what a session is costing.

+
manage: ak setup
+
+
+
⇆ + Dual-host routing & bridgeconfigured
+

With both Claude and Codex enabled, each kind of work is routed to the host + that's best at it — and each host can hand a subtask to the other mid-flight.

+
manage: ak host pick
+
+
+
⚙ + Background daemonconfigured
+

Local-only workers that keep learning and maintenance ticking between + sessions. Free by default — anything that would spend AI tokens stays strictly opt-in.

+
manage: ruflo daemon
+
+
+
✓ + Permission allowlistconfigured
+

Pre-approved command patterns for the stack's own tools, so they don't + interrupt you with permission prompts. Every rule is disclosed at setup, none added + silently.

+
manage: ak setup
+
+
+
Design note: configured surfaces get the same card shape as packages — + one grammar to learn — but a distinct configured chip, no version, and a + manage: line instead of source links, because "where do I change + this" is their equivalent of "where do I read more".
+
+ + +
+ Design +

Anatomy of a card

+
+
+ 1 + 2 + 3 + 4 + 5 +
+
+ rf + rufloinstalled · v3.34.0 +
+ The orchestration engine underneath. +

Gives your agents what a single session can't: memory that survives + restarts, teams of specialists working in parallel…

+
GitHub ↗ + npm ↗
+
+
+
+
1Icon tile. Official mark where one + genuinely exists (the three hosts); otherwise a monogram on the category hue. Never an + invented logo.
+
2Name + state chip. The only place + runtime facts appear — fed by detection the dashboard already does, never by the prose. + Absent reads "not installed — ak setup adds it"; a failed status fetch reads "state + unknown" while the card still renders.
+
3Tagline. Ten plain words or fewer — + the card's thesis, scannable on its own.
+
4One paragraph, ~50 words. What it does + for you, active voice, every term of art explained where it's used. Depth lives + behind the Docs link, not here.
+
5Link pills. GitHub / npm / Docs — the + receipts. Plain outbound anchors the user clicks; the kit itself never fetches them.
+
+
+
+ +
ak · About — design mock for ADR-0026 · illustrative content · both themes via the dashboard's own tokens
+
+ + diff --git a/docs/ddd/component-directory.md b/docs/ddd/component-directory.md new file mode 100644 index 0000000..9a38fc7 --- /dev/null +++ b/docs/ddd/component-directory.md @@ -0,0 +1,169 @@ +# Component Directory Domain + +> **Draft for review.** This document specifies the domain proposed by +> [ADR-0026](../adr/0026-about-component-directory.md). Nothing described here is implemented +> yet; module paths named below are the intended homes, not existing files. On acceptance, the +> new terms merge into [Ubiquitous language](ubiquitous-language.md) and the context joins the +> [context map](context-map.md). + +## Purpose + +Component directory answers a new user's first question — **"what did agentic-kit put on my +machine, and why is each thing worth having?"** — inside the dashboard's proposed **About** +primary area, placed first in the tab order. It owns the curated editorial identity of every +component the kit installs or configures: a plain-language tagline, one friendly paragraph of +value proposition, outbound links to the source (GitHub), package (npm), and public docs, an +icon, a category, and a deliberate reading order. It joins that editorial content, at render +time, with detection facts the dashboard already has — installed or not, version, install +method — and never collects anything of its own. + +The shared terms in [Ubiquitous language](ubiquitous-language.md) are normative; the +[terms table](#ubiquitous-language-additions) below is this draft's proposed addition to it. + +## Why this is a separate context + +The neighboring contexts each own a different kind of fact about the same components; this one +owns the only kind none of them can: **editorial identity**. + +- **[Integration management](integration-management.md)** owns capability registries, bindings, + and ownership — what a component *can do* and what ak *manages about it*. It has no voice: no + value proposition, no links for a human, no reading order. The directory borrows its registry + as a parity gate (every managed tool must have an entry) and borrows nothing else. +- **Overview / status** owns health verdicts. The directory renders a version chip from the + same detection facts, but a card is an introduction, not a verdict — About never says + "degraded," it says what the thing is for and where to read more. +- **[Machine footprint](machine-footprint.md)** ([ADR-0025](../adr/0025-machine-footprint-metrics.md)) + owns measured cost. The directory renders no numbers beyond version strings — bytes, counts, + and processes belong there. +- **Documentation** (`docs/*.md`) explains ak to maintainers in reference register. The + directory is the new-user register, structured as data (per-component entries) so + completeness is testable, which prose cannot be. + +The editorial/detection split is the load-bearing boundary: editorial content is **authored, +versioned, and reviewed** with the release; detection facts are **observed at render**. A card +can therefore never claim runtime state in prose — the chip says `installed v3.34.0` because +detection said so, while the paragraph would read identically on a machine where the tool is +absent (with the chip honestly reading `not installed — ak setup adds it`). + +## Model + +```text +src/lib/dashboard/about-directory.mjs (intended home; pure data + tiny accessors) + +DirectoryEntry { + id, // stable key; the managed-tools registry key where one exists + category, // 'hosts' | 'engine-memory' | 'quality' | 'safety' | 'knowledge' + // | 'kit' | 'configured' + name, tagline, // tagline: plain-language, ≤ 10 words + paragraph, // one paragraph, ~50 words, new-user register (see contract below) + links: [ { kind: 'github'|'npm'|'docs', label, url } ], // https only + icon: { kind: 'official'|'monogram', ref }, // official = the already-shipped host marks + detectionKey? // join key into existing status/managed-tools facts; absent for + // configured surfaces, which join on their subsystem row instead +} + +directoryEntries() -> DirectoryEntry[] (curated order: hosts → engine & memory → + quality → safety → knowledge → kit → configured) + + + joined at render with existing facts (no new collection): +/api/status rows + managed-tools detection -> { installed, version, installMethod } + + | + v +About primary area (leftmost tab): hero orientation strip + category sections of cards + Card = icon tile · name + state chip · tagline · paragraph · link pills +``` + +### The editorial register contract + +The paragraph and tagline are covered by an explicit writing contract, enforced in review (and +lintable where mechanical): + +1. **One paragraph, ~50 words, no more.** A card is an introduction, not documentation — the + Docs link is where depth lives. +2. **Plain language.** Every term of art is either avoided or explained in the sentence using + it. "Agent swarms" renders as "teams of specialist agents"; "MCP" as "the plug-in protocol + agents use to reach tools." +3. **Value stated as what it does for the user**, in active voice — never marketing + superlatives, never claims about quality ("best," "blazing") that detection can't back. +4. **No runtime claims in prose.** "Installed," "running," "healthy" are chip words, fed by + detection; prose describes purpose, which is true whether or not the tool is present. +5. **Friendly ≠ cute.** Warmth comes from clarity and usefulness; no exclamation marks doing + the work of substance. + +### Iconography + +Official marks are used only where the dashboard already ships them as official — the three +host SVGs Observability renders on session rows — and are reused byte-identically so a +component looks the same everywhere. Every other component gets a **monogram tile**: its +initial(s) on a rounded tile in its category hue. A monogram is an honest "no official mark" +statement, not a stand-in logo; if an upstream later publishes a usable mark, swapping it in is +a one-entry change. All icons are inline (SVG or styled text) — the page stays self-contained +under the dashboard's CSP. + +### Configured surfaces + +Non-package things ak sets up — MCP registrations, managed guidance blocks +(CLAUDE.md/AGENTS.md), statuslines, the dual-host routing policy and Claude↔Codex bridge, the +background daemon, permission allowlists — are directory entries too, in the `configured` +category. Their chip reads `configured` (joined from the relevant status subsystem row), their +paragraph explains what was configured and why it helps, and each names the command that +manages it (`ak host`, `ak sync`, `ak x mcp pick`) so "yours to change" is actionable, not a +platitude. + +## Delivery + +No new endpoint. The directory module is imported by the page/client the same way `groups.mjs` +already is, and the state chips join client-side against the `/api/status` payload the +dashboard polls anyway. A status fetch that fails degrades every chip to `state unknown` while +the editorial content — which needs no network and no facts — renders in full: the page's +purpose survives its join's absence ([ADR-0023](../adr/0023-fail-closed-operations-and-explicit-degradation.md) +honesty, applied to a page whose primary content is static). + +Outbound links open the user's browser; the kit performs no egress. The directory's URLs are +swept by the same nightly external link check that covers `docs/**` — a rotted link is CI red, +not a permanent dead end. + +## Invariants + +1. **Editorial content is authored and versioned with the release** — checked in, reviewed, + never generated or fetched at runtime. +2. **Detection facts come only from existing collectors.** The directory probes nothing, adds + no endpoint, and a failed status join degrades chips to `unknown` without hiding cards. +3. **Prose never claims runtime state.** Installed/version/configured render exclusively as + chips fed by detection; the paragraph reads true on any machine. +4. **Registry↔directory parity is a test.** Every managed tool has exactly one entry; no entry + exists for something ak neither installs nor configures. +5. **Links are `https`, named-host, user-initiated**; the kit fetches none of them; all are + covered by the nightly external link sweep. +6. **Official marks only where genuinely official and already shipped**; everything else is an + explicit monogram tile. No fabricated brand assets, ever. +7. **No numbers beyond version strings.** Health, cost, counts, and activity belong to their + own contexts; About introduces, it does not measure. +8. **The register contract governs every entry** (length, plain language, active voice, no + superlatives, no runtime claims). +9. **Order is curated and stable** — hosts first, the kit's own card near the end, configured + surfaces last — never derived from popularity, size, or health. +10. **One icon per component, everywhere.** The directory's icon spec is the single source for + that component's mark across the dashboard. + +## Ubiquitous language additions + +| Term | Meaning | +|------|---------| +| Component directory | The curated catalog of everything ak installs or configures, with editorial identity per entry | +| DirectoryEntry | One component's editorial identity: category, tagline, paragraph, links, icon, and a detection join key | +| Editorial content | Authored, versioned prose and links — the part of a card that is true regardless of machine state | +| Detection fact | An observed install/version/configured fact borrowed read-only from existing collectors, rendered only as chips | +| State chip | The card element that renders detection facts (`installed v…` / `not installed — ak setup adds it` / `configured` / `unknown`) | +| Monogram tile | The honest icon for a component with no official mark: initials on a category-hued tile | +| Register contract | The editorial writing rules (one ~50-word paragraph, plain language, no runtime claims, no superlatives) | +| Parity gate | The test asserting managed-tools registry ↔ directory completeness in both directions | + +## References + +- [ADR-0026](../adr/0026-about-component-directory.md) +- [Integration management](integration-management.md) — the registry this stays in parity with +- [Machine footprint](machine-footprint.md) — the measurement context this deliberately isn't +- [Context map](context-map.md) — joined on acceptance +- [Dashboard guide](../DASHBOARD.md) From f15ba90b744634ca090f87ad230fc1b7484baa12 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 16:31:39 -0700 Subject: [PATCH 05/19] feat(about): component directory, ak about, and a registry parity gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements ADR-0026. A curated directory module carries each component's editorial identity — tagline, one plain-language paragraph, source/npm/docs links, icon, category — rewritten from verified upstream sources rather than assumption. It collects nothing: state chips join client-side against the status payload the dashboard already polls, so prose never claims runtime state and a failed join degrades chips to unknown while content still renders. The parity test is the point: every managed tool must have exactly one entry and no entry may exist for something ak neither installs nor configures, so a future tool cannot ship without its card. --- src/commands/about.mjs | 296 +++++++++++++++++ src/lib/dashboard/about-directory.mjs | 343 ++++++++++++++++++++ tests/kit/about-directory.test.mjs | 449 ++++++++++++++++++++++++++ 3 files changed, 1088 insertions(+) create mode 100644 src/commands/about.mjs create mode 100644 src/lib/dashboard/about-directory.mjs create mode 100644 tests/kit/about-directory.test.mjs diff --git a/src/commands/about.mjs b/src/commands/about.mjs new file mode 100644 index 0000000..9d1fb8b --- /dev/null +++ b/src/commands/about.mjs @@ -0,0 +1,296 @@ +// ak about — the component directory in the terminal (ADR-0026). +// +// The CLI twin of the dashboard's About area, reading the SAME frozen editorial +// data (src/lib/dashboard/about-directory.mjs). There is deliberately no second +// copy of the copy: a paragraph reviewed for the release must read identically +// in both surfaces, or one of them shipped text nobody signed off on. +// +// The editorial/detection split survives the port. Prose comes from the +// directory and never claims runtime state; the state chip is the only place a +// runtime fact appears, and it is fed by collectors that ALREADY EXIST — the +// version primitives `ak status` itself calls, and `ak status`'s own rows for +// the configured surfaces. This command adds no probe of its own. When a +// detection source fails the chip degrades to `state unknown — ` and the +// entry still renders: an unmeasured component is never drawn as absent +// (ADR-0023 / component-directory invariants 2 and 3). +import { heading, dim, bold, glyph, green, yellow } from '../lib/output.mjs'; +import { CATEGORY_ORDER, directoryEntries } from '../lib/dashboard/about-directory.mjs'; + +export const options = { + json: { type: 'boolean', default: false }, + category: { type: 'string' }, + 'no-detect': { type: 'boolean', default: false }, +}; + +export const help = `ak about — what agentic-kit installs and configures, and why + +One entry per component: what it is, what it does for you, where to read more, +and an honest state chip. Prose is authored with the release; the chip is the +only runtime fact, read from the same detection \`ak status\` uses. The dashboard's +About tab renders this identical directory. + +Usage: + ak about [entry-id] + +Options: + --category only one category: ${CATEGORY_ORDER.join(', ')} + --no-detect editorial only — resolve no state chips at all (instant) + --json emit the directory plus resolved state chips + +Examples: + ak about the whole directory, grouped by category + ak about ruflo one entry + ak about --category hosts just the agent CLIs + ak about --no-detect the authored copy alone, no detection at all + ak about --json machine-readable directory + state`; + +// The directory exports no category headings on purpose: section copy belongs to +// the page that renders it, not to the shared data. These are the terminal's. +const CATEGORY_LABELS = Object.freeze({ + hosts: 'Hosts — the agents you talk to', + 'engine-memory': 'Engine and memory — what runs underneath', + quality: 'Quality — proving the work is done', + safety: 'Safety — checking untrusted text', + knowledge: 'Knowledge — grounded answers about this stack', + kit: 'The kit itself', + configured: 'Configured for you — surfaces ak sets up', +}); + +const installed = (version, note = null) => ({ state: 'installed', version: version ?? null, note }); +const absent = () => ({ state: 'absent', version: null, note: null }); +const configured = () => ({ state: 'configured', version: null, note: null }); +const attention = (note) => ({ state: 'attention', version: null, note: String(note) }); +const unknown = (why) => ({ state: 'unknown', version: null, note: String(why) }); +// Distinct from `unknown`: nothing failed here, the user asked for no detection. +// Sharing the chip would flag all fifteen entries as if something were wrong. +const skipped = () => ({ state: 'skipped', version: null, note: 'detection skipped (--no-detect)' }); + +const reasonOf = (error) => String(error?.message ?? error); + +/** + * State chips for the packaged entries, from the primitives `ak status` already + * calls. Each source is guarded independently so one unavailable collector + * degrades one chip, never the page. Network-free by construction: every call + * here reads the filesystem or resolves a binary — the drift/latest-version + * lookups that DO reach npm are deliberately not consulted, because "is it + * installed" is a local question and About asks nothing else. + * + * @param {{ pkgRoot?: string }} input + * @returns {Promise>} + */ +async function detectPackaged({ pkgRoot }) { + const states = new Map(); + + try { + const { HOSTS, hostInstallState } = await import('../lib/providers.mjs'); + for (const host of HOSTS) { + try { + const state = await hostInstallState(host); + states.set(`hosts.${host.id}`, state.method === 'absent' + ? absent() + // An externally-installed host (mise, brew, a native installer) is + // present and ak says so — while naming the owner, because ak does + // not manage its updates (MANAGED-TOOLS "honest disowning"). + : installed(state.version, + state.method === 'external' ? 'external install — self-managed' : null)); + } catch (error) { + states.set(`hosts.${host.id}`, unknown(reasonOf(error))); + } + } + } catch (error) { + for (const id of ['claude', 'codex', 'opencode']) states.set(`hosts.${id}`, unknown(reasonOf(error))); + } + + const { installedVersion, KIT_PKG } = await import('../lib/versions.mjs'); + for (const [key, pkg] of [['ruflo', 'ruflo'], ['agentic-qe', 'agentic-qe']]) { + try { + const version = installedVersion(pkg); + states.set(key, version ? installed(version) : absent()); + } catch (error) { + states.set(key, unknown(reasonOf(error))); + } + } + + try { + const { coherence } = await import('../lib/agentdb.mjs'); + const c = coherence(); + states.set('agentdb', c.present ? installed(c.global) : absent()); + } catch (error) { + states.set('agentdb', unknown(reasonOf(error))); + } + + // aidefence and @claude-flow/security are nested under global ruflo, not + // global packages, so there is no version to read — presence is the whole + // fact ak has. Reporting the pair separately matters: security-without- + // aidefence is the state in which `security defend` silently does nothing. + try { + const { aidefencePresent, securityPresent } = await import('../lib/natives.mjs'); + const security = securityPresent(); + if (security && aidefencePresent()) states.set('security', installed(null)); + else if (security) states.set('security', attention('@claude-flow/security present, aidefence missing')); + else states.set('security', absent()); + } catch (error) { + states.set('security', unknown(reasonOf(error))); + } + + try { + const brain = await import('../lib/ruvnet-brain.mjs'); + states.set('ruvnet-brain', brain.present() ? installed(brain.installedVersion()) : absent()); + } catch (error) { + states.set('ruvnet-brain', unknown(reasonOf(error))); + } + + // The kit's own version: the globally installed copy when there is one, else + // the running checkout's manifest — the same order collectInstall resolves + // `self` in, so a linked dev install reports a version rather than "absent". + try { + let version = installedVersion(KIT_PKG); + if (!version && pkgRoot) { + const [{ readFileSync }, { default: path }] = await Promise.all([ + import('node:fs'), import('node:path'), + ]); + version = JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')).version ?? null; + } + states.set('self', version ? installed(version) : absent()); + } catch (error) { + states.set('self', unknown(reasonOf(error))); + } + + return states; +} + +/** + * State chips for the configured surfaces, joined from `ak status` rows exactly + * as the dashboard joins them from /api/status. A subsystem that emits NO row is + * unknown, never `configured`: an unjoined key is an unmeasured fact, not a + * satisfied one (the permission allowlist is in that position today). + */ +function detectConfigured(rows) { + const bySubsystem = new Map(); + for (const row of rows) { + if (!bySubsystem.has(row.subsystem)) bySubsystem.set(row.subsystem, []); + bySubsystem.get(row.subsystem).push(row); + } + return (subsystem) => { + const matched = bySubsystem.get(subsystem); + if (!matched?.length) return unknown(`ak status emits no '${subsystem}' row`); + const bad = matched.find((row) => row.level === 'fail') ?? matched.find((row) => row.level === 'warn'); + return bad ? attention(bad.message) : configured(); + }; +} + +/** The chip's rendered text. `installed` without a version is a real state — + * some components are nested packages with no manifest ak may read. */ +function chipText(state) { + if (state.state === 'installed') return state.version ? `installed · v${state.version}` : 'installed'; + if (state.state === 'absent') return 'not installed — ak setup adds it'; + if (state.state === 'configured') return 'configured'; + if (state.state === 'attention') return 'needs attention'; + if (state.state === 'skipped') return 'state not resolved — detection skipped'; + return `state unknown — ${state.note}`; +} + +const chipLevel = (state) => (state.state === 'installed' || state.state === 'configured' ? 'ok' + : state.state === 'attention' || state.state === 'unknown' ? 'warn' : 'none'); + +function paint(state, text) { + const level = chipLevel(state); + if (level === 'ok') return green(text); + if (level === 'warn') return yellow(text); + return dim(text); +} + +/** Greedy wrap. Long tokens (URLs) are never broken — a split URL is unusable. */ +function wrap(text, width) { + const lines = []; + let line = ''; + for (const word of String(text).split(/\s+/).filter(Boolean)) { + if (line && line.length + 1 + word.length > width) { lines.push(line); line = word; } + else line = line ? `${line} ${word}` : word; + } + if (line) lines.push(line); + return lines; +} + +function renderEntry(entry, state, width) { + const indent = ' '; + const body = Math.max(40, width - indent.length); + console.log(` ${glyph(chipLevel(state))} ${bold(entry.name)} ${paint(state, chipText(state))}`); + console.log(`${indent}${dim(entry.tagline)}`); + for (const line of wrap(entry.paragraph, body)) console.log(`${indent}${line}`); + for (const link of entry.links) console.log(`${indent}${dim(link.label.padEnd(6))} ${link.url}`); + if (entry.manage) console.log(`${indent}${dim('manage'.padEnd(6))} ${entry.manage}`); + console.log(''); +} + +/** + * @param {{ flags: Record, positionals: string[], pkgRoot?: string, + * deps?: { collectStatus?: Function, cwd?: string } }} input + */ +export async function run({ flags, positionals, pkgRoot, deps = {} }) { + const wanted = positionals[0]; + // The directory is a frozen literal, so its inferred type is a union of 15 + // distinct shapes and `entry.subsystem` is only on some of them. Widening here + // keeps the field access honest at runtime (a missing key is `undefined`, + // which is exactly the packaged/configured discriminator) without weakening + // any type the directory itself publishes. + /** @type {Array>} */ + const entries = directoryEntries().filter((entry) => ( + (!wanted || entry.id === wanted) && (!flags.category || entry.category === flags.category) + )); + + if (!entries.length) { + const what = [wanted && `id '${wanted}'`, flags.category && `category '${flags.category}'`] + .filter(Boolean).join(' and '); + console.log(`${glyph('warn')} no directory entry matches ${what}`); + console.log(dim(`ids: ${directoryEntries().map((entry) => entry.id).join(', ')}`)); + return 2; + } + + const states = new Map(); + if (flags['no-detect']) { + for (const entry of entries) states.set(entry.id, skipped()); + } else { + const packaged = await detectPackaged({ pkgRoot }); + // One `ak status` collect for every configured chip. Skipped entirely when + // no configured entry survived the filter, so `ak about ruflo` stays cheap. + /** @type {(subsystem: string) => { state: string, version: string|null, note: string|null }} */ + let configuredState = () => unknown('status collection not run'); + if (entries.some((entry) => entry.subsystem)) { + try { + const collectStatus = deps.collectStatus + ?? (await import('./status.mjs')).collect; + configuredState = detectConfigured(await collectStatus({ pkgRoot, cwd: deps.cwd ?? process.cwd() })); + } catch (error) { + const why = reasonOf(error); + configuredState = () => unknown(why); + } + } + for (const entry of entries) { + states.set(entry.id, entry.subsystem + ? configuredState(entry.subsystem) + : packaged.get(entry.detectionKey) ?? unknown(`no detection source for '${entry.detectionKey}'`)); + } + } + + if (flags.json) { + console.log(JSON.stringify({ + detection: !flags['no-detect'], + entries: entries.map((entry) => ({ ...entry, state: states.get(entry.id) })), + }, null, 2)); + return 0; + } + + const width = Math.max(60, Math.min(100, process.stdout.columns || 80)); + heading('ak about — what agentic-kit installs and configures'); + console.log(dim(' purpose is authored; presence is measured — only the chip claims runtime state')); + + for (const category of CATEGORY_ORDER) { + const inCategory = entries.filter((entry) => entry.category === category); + if (!inCategory.length) continue; + heading(CATEGORY_LABELS[category] ?? category); + console.log(''); + for (const entry of inCategory) renderEntry(entry, states.get(entry.id), width); + } + return 0; +} diff --git a/src/lib/dashboard/about-directory.mjs b/src/lib/dashboard/about-directory.mjs new file mode 100644 index 0000000..fd7c321 --- /dev/null +++ b/src/lib/dashboard/about-directory.mjs @@ -0,0 +1,343 @@ +// The About area's component directory: the authored editorial identity of every +// component agentic-kit installs or configures (ADR-0026 / docs/ddd/component-directory.md). +// +// THE EDITORIAL/DETECTION SPLIT is why this module exists and why it is pure data. +// Editorial content — tagline, paragraph, links, icon, category, order — is authored, +// reviewed, and versioned WITH THE RELEASE. Detection facts — installed, version, +// install method, configured — are observed at render by collectors that already exist +// (/api/status rows, the drift array, the managed-tools machinery). The directory owns +// only the first kind and joins the second client-side; it probes nothing, fetches +// nothing, and adds no endpoint. Importing this module must therefore have zero side +// effects so a unit test can assert the whole catalog without a machine to measure. +// +// PROSE NEVER CLAIMS RUNTIME STATE. "Installed", "running", "healthy" are chip words fed +// by detection. Every paragraph below reads true on a machine where the component is +// absent — that is the structural guarantee behind the state chip, not a style +// preference: if copy could assert presence, a stale sentence would contradict an honest +// chip on the same card. Purpose is timeless; presence is measured. +// +// The register contract governing this copy (one paragraph, ~50 words, plain language, +// active voice, no superlatives, no runtime claims) is specified in the DDD doc. + +/** Curated, stable reading order. Never derived from popularity, size, or health: + * the things a new user already recognizes come first, infrastructure after, and the + * kit's own card sits near the end because it is the caretaker, not the point. */ +export const CATEGORY_ORDER = Object.freeze([ + 'hosts', 'engine-memory', 'quality', 'safety', 'knowledge', 'kit', 'configured', +]); + +// Official marks are reused BYTE-IDENTICALLY from the marks the dashboard already ships +// for Observability's session rows (`hostIcon()` in live/client.mjs, `sourceHostIcon()` +// in client.mjs), which dispatch on these exact host ids. Only the three hosts have a +// genuinely official, already-shipped mark; everything else is an explicit monogram +// tile — initials on a category hue — because a fabricated logo would misrepresent a +// project we do not speak for. `hue` names a CSS custom property the About stylesheet +// defines; the directory names the token, it does not pick the colour. +const OFFICIAL = (ref) => Object.freeze({ kind: 'official', ref }); +const MONOGRAM = (ref, hue) => Object.freeze({ kind: 'monogram', ref, hue }); + +const link = (kind, label, url) => Object.freeze({ kind, label, url }); + +// detectionKey joins a packaged entry to the managed-tools detection facts. The keys are +// heterogeneous because the facts are: hosts resolve through the host registry +// (`hosts.`), agentdb / agentic-qe / ruvnet-brain have their own `ak status` +// subsystem row, aidefence is reported by the `security` row, and the kit reports as +// `self`. `npmPackage` is carried alongside because the version chip's other source — +// the drift array — is keyed by package name, and deriving that from a link URL would be +// a parsing trick rather than a stated fact. Configured surfaces have no detectionKey: +// they are not packages, so they carry the status `subsystem` row their chip joins on +// plus the `manage` command that changes them. `subsystem` names the row a chip SHOULD +// join; where no such row is emitted yet the chip must degrade to `unknown` rather than +// assume `configured` — an unjoined key is an unmeasured fact, not a satisfied one. +const ENTRIES = Object.freeze([ + Object.freeze({ + id: 'claude-code', + category: 'hosts', + name: 'Claude Code', + tagline: "Anthropic's coding agent, in your terminal.", + paragraph: + 'Point it at a repository and talk to it: it reads your code, proposes and makes ' + + 'changes, runs your tests, and explains each step as it goes. It is one of the ' + + 'agent CLIs — coding assistants that live in your terminal — the rest of this ' + + 'toolkit exists to strengthen.', + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/anthropics/claude-code'), + link('npm', 'npm', 'https://www.npmjs.com/package/@anthropic-ai/claude-code'), + link('docs', 'Docs', 'https://docs.claude.com/en/docs/claude-code/overview'), + ]), + icon: OFFICIAL('claude'), + detectionKey: 'hosts.claude', + npmPackage: '@anthropic-ai/claude-code', + }), + Object.freeze({ + id: 'codex', + category: 'hosts', + name: 'Codex', + tagline: "OpenAI's coding agent — a second pair of eyes.", + paragraph: + "OpenAI's take on the same idea: an agent that works alongside you in the terminal " + + 'on real repositories. When both it and Claude Code are enabled, agentic-kit runs ' + + "them as peers — either can lead a job, review the other's output, or pick up a " + + 'step the other stalled on.', + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/openai/codex'), + link('npm', 'npm', 'https://www.npmjs.com/package/@openai/codex'), + link('docs', 'Docs', 'https://developers.openai.com/codex/cli'), + ]), + icon: OFFICIAL('codex'), + detectionKey: 'hosts.codex', + npmPackage: '@openai/codex', + }), + Object.freeze({ + id: 'opencode', + category: 'hosts', + name: 'OpenCode', + tagline: 'An open-source coding agent, run as a worker.', + paragraph: + 'A community-built agent CLI from the OpenCode project. agentic-kit treats it as an ' + + 'opt-in worker: you route particular jobs to it by name, its sessions appear ' + + 'alongside the others in Observability, and it never takes the lead on a job ' + + 'unless you say so.', + links: Object.freeze([ + // The upstream repository moved from sst/opencode; github.com/sst/opencode still + // resolves only by redirect, so the canonical owner is named here directly. + link('github', 'GitHub', 'https://github.com/anomalyco/opencode'), + link('npm', 'npm', 'https://www.npmjs.com/package/opencode-ai'), + link('docs', 'Docs', 'https://opencode.ai/docs/'), + ]), + icon: OFFICIAL('opencode'), + detectionKey: 'hosts.opencode', + npmPackage: 'opencode-ai', + }), + Object.freeze({ + id: 'ruflo', + category: 'engine-memory', + name: 'ruflo', + tagline: 'The orchestration engine under your agents.', + paragraph: + 'Gives your agents what a single session cannot: decisions remembered across ' + + "restarts, teams of specialist agents working one task in parallel (a 'swarm'), " + + 'hooks that send each job to the agent suited to it, and security checks along ' + + 'the way. agentic-kit installs it and keeps it healthy.', + links: Object.freeze([ + // The upstream repository was renamed from ruvnet/claude-flow; npm still records + // the old URL, so the canonical name is stated here rather than derived. + link('github', 'GitHub', 'https://github.com/ruvnet/ruflo'), + link('npm', 'npm', 'https://www.npmjs.com/package/ruflo'), + link('docs', 'Docs', 'https://github.com/ruvnet/ruflo/tree/main/docs'), + ]), + icon: MONOGRAM('rf', '--hue-engine'), + detectionKey: 'ruflo', + npmPackage: 'ruflo', + }), + Object.freeze({ + id: 'agentdb', + category: 'engine-memory', + name: 'agentdb', + tagline: 'Where what your agents learn is stored.', + paragraph: + 'The store behind that memory: a single local file holding what agents recorded — ' + + 'decisions, state, and the reasons behind them — searchable by meaning as well as ' + + 'by keyword, with the links between entries kept too. agentic-kit pins its version ' + + 'to the one ruflo ships, so the two cannot drift apart.', + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/ruvnet/agentdb'), + link('npm', 'npm', 'https://www.npmjs.com/package/agentdb'), + link('docs', 'Docs', 'https://github.com/ruvnet/agentdb/tree/main/docs'), + ]), + icon: MONOGRAM('db', '--hue-engine'), + detectionKey: 'agentdb', + npmPackage: 'agentdb', + }), + Object.freeze({ + id: 'agentic-qe', + category: 'quality', + name: 'agentic-qe', + tagline: 'A quality-engineering fleet you can call on.', + paragraph: + "Specialist agents for the unglamorous half of shipping: writing tests in your " + + "project's own framework, finding which untested code carries the most risk, " + + 'spotting tests that pass and fail at random, and gating a change on those ' + + "results. 'The agent says it works' becomes something you can check.", + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/proffesor-for-testing/agentic-qe'), + link('npm', 'npm', 'https://www.npmjs.com/package/agentic-qe'), + link('docs', 'Docs', 'https://github.com/proffesor-for-testing/agentic-qe#readme'), + ]), + icon: MONOGRAM('qe', '--hue-quality'), + detectionKey: 'agentic-qe', + npmPackage: 'agentic-qe', + }), + Object.freeze({ + id: 'aidefence', + category: 'safety', + name: 'aidefence + security', + tagline: 'Checks untrusted text before your agents act on it.', + paragraph: + "A pasted issue, a fetched web page, a stranger's README — any of them can carry " + + "instructions aimed at your agent rather than at you ('prompt injection'). " + + 'aidefence scans text for those patterns, for jailbreak attempts, and for exposed ' + + 'personal data; @claude-flow/security supplies the primitives ruflo\'s security ' + + 'commands are built on.', + links: Object.freeze([ + link('github', 'GitHub', + 'https://github.com/ruvnet/ruflo/tree/main/v3/@claude-flow/aidefence'), + link('npm', 'npm', 'https://www.npmjs.com/package/@claude-flow/aidefence'), + // No upstream doc site exists for these two packages; ak's own troubleshooting + // page is the honest "where to read more", not a stand-in for one. + link('docs', 'Docs', + 'https://github.com/pacphi/agentic-kit/blob/main/docs/TROUBLESHOOTING.md'), + ]), + icon: MONOGRAM('ad', '--hue-safety'), + detectionKey: 'security', + npmPackage: '@claude-flow/aidefence', + }), + Object.freeze({ + id: 'ruvnet-brain', + category: 'knowledge', + name: 'RuvNet Brain', + tagline: 'Answers about this stack, from its real source.', + paragraph: + "These tools change faster than any model's training data, so an agent asked about " + + 'them tends to guess. The Brain is a local knowledge base built from those ' + + "projects' real source, and it answers with the file path each answer came from — " + + 'so you can check the claim yourself.', + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/stuinfla/ruvnet-brain'), + link('npm', 'npm', 'https://www.npmjs.com/package/ruvnet-brain'), + link('docs', 'Docs', 'https://isovision.ai/ruvnet-brain/'), + ]), + icon: MONOGRAM('B', '--hue-knowledge'), + detectionKey: 'ruvnet-brain', + npmPackage: 'ruvnet-brain', + }), + Object.freeze({ + id: 'agentic-kit', + category: 'kit', + name: 'agentic-kit', + tagline: 'The caretaker for everything above.', + paragraph: + 'One command sets this collection up, one command repairs it after an upgrade moves ' + + 'something, and this dashboard shows what all of it is doing. When a piece drifts ' + + 'from the version it should be on, breaks, or disappears, ak names the problem and ' + + 'the command that fixes it.', + links: Object.freeze([ + link('github', 'GitHub', 'https://github.com/pacphi/agentic-kit'), + link('npm', 'npm', 'https://www.npmjs.com/package/@pacphi/agentic-kit'), + link('docs', 'Docs', 'https://github.com/pacphi/agentic-kit/tree/main/docs'), + ]), + icon: MONOGRAM('ak', '--hue-kit'), + detectionKey: 'self', + npmPackage: '@pacphi/agentic-kit', + }), + Object.freeze({ + id: 'mcp-registrations', + category: 'configured', + name: 'MCP registrations', + tagline: 'Your tools, plugged in once for every project.', + paragraph: + 'Agents reach outside tools through MCP, the plug-in protocol for AI tooling. ak ' + + "registers this stack's MCP servers once at your user level, so every repository " + + 'you open gets them without per-project setup, and lets you switch off tool groups ' + + 'you would rather not have. Run `ak x mcp pick` to change it.', + links: Object.freeze([]), + icon: MONOGRAM('M', '--info'), + subsystem: 'mcp', + manage: 'ak x mcp pick', + }), + Object.freeze({ + id: 'guidance-blocks', + category: 'configured', + name: 'Guidance blocks', + tagline: 'House rules your agents read at startup.', + paragraph: + 'Short managed sections inside CLAUDE.md and AGENTS.md that tell each agent how this ' + + 'stack fits together and which tool to reach for. ak writes only between its own ' + + 'markers, so anything you wrote stays untouched, and refreshes the text as the ' + + 'tools change. Run `ak sync` to reapply it.', + links: Object.freeze([]), + icon: MONOGRAM('G', '--info'), + subsystem: 'blocks', + manage: 'ak sync', + }), + Object.freeze({ + id: 'statuslines', + category: 'configured', + name: 'Statuslines', + tagline: 'A live footer showing what a session costs.', + paragraph: + 'A footer line inside Claude Code and Codex showing the model in use, what the ' + + "session has spent so far, and how much of your plan's limit is left — so cost " + + 'stays visible while you work instead of arriving later. Run `ak sync` when an ' + + 'upgrade wipes it.', + links: Object.freeze([]), + icon: MONOGRAM('S', '--info'), + subsystem: 'statusline', + manage: 'ak sync', + }), + Object.freeze({ + id: 'dual-host-routing', + category: 'configured', + name: 'Dual-host routing & bridge', + tagline: 'Each kind of work goes to the better host.', + paragraph: + 'With Claude Code and Codex both enabled, ak records which host handles which kind ' + + 'of work — coding, testing, review, security — and wires each one to reach the ' + + 'other as a tool, so a job can be handed across mid-flight. You pick the table and ' + + 'which host leads: `ak host pick`.', + links: Object.freeze([]), + icon: MONOGRAM('R', '--info'), + subsystem: 'routing', + manage: 'ak host pick', + }), + Object.freeze({ + id: 'background-daemon', + category: 'configured', + name: 'Background daemon', + tagline: 'Local workers that keep things tidy between sessions.', + paragraph: + 'A background process ruflo runs between your sessions for upkeep — learning from ' + + 'finished work, tidying up — staffed by local workers that cost nothing. Anything ' + + 'that would spend money on a model stays opt-in, and each daemon expires on its ' + + 'own. List or stop them with `ak x daemon-gc`.', + links: Object.freeze([]), + icon: MONOGRAM('D', '--info'), + subsystem: 'daemons', + manage: 'ak x daemon-gc', + }), + Object.freeze({ + id: 'permission-allowlist', + category: 'configured', + name: 'Permission allowlist', + tagline: 'Fewer prompts for the tools you already trust.', + paragraph: + "Pre-approved command patterns for this stack's own tools, so routine calls stop " + + 'asking your permission every time. Every rule is shown to you during setup and ' + + "written into your project's settings file, and ak strips out any rule it finds " + + 'there that it never disclosed. Run `ak setup` to review them.', + links: Object.freeze([]), + icon: MONOGRAM('P', '--info'), + // `ak status` emits no `permissions` row today; until one exists this chip reads + // unknown. Naming the row it would join is the honest placeholder — inventing a + // green `configured` for an unmeasured surface is the failure this avoids. + subsystem: 'permissions', + manage: 'ak setup', + }), +]); + +/** The whole directory in curated order. Frozen: a card renderer reads it, a parity test + * asserts over it, and neither may edit the release's authored copy in place. */ +export function directoryEntries() { + return ENTRIES; +} + +/** One entry by its stable id, or null — never a partial object, so a caller that + * mistypes an id gets an obvious absence rather than a card with empty prose. */ +export function entryById(id) { + return ENTRIES.find((e) => e.id === id) || null; +} + +/** Entries in one category, in curated order; an unknown category yields []. */ +export function entriesByCategory(category) { + return ENTRIES.filter((e) => e.category === category); +} diff --git a/tests/kit/about-directory.test.mjs b/tests/kit/about-directory.test.mjs new file mode 100644 index 0000000..8a534b5 --- /dev/null +++ b/tests/kit/about-directory.test.mjs @@ -0,0 +1,449 @@ +// about-directory.test.mjs — the gate ADR-0026 §3 promises: "completeness is a gate, not a +// hope". The About area's cards are authored copy, and authored copy is exactly the kind of +// thing that quietly stops matching reality — a tool gets added to the kit, ships, and never +// grows a card, or a card outlives the tool it introduces. Review cannot catch that reliably; +// a test can. +// +// The parity gate below therefore reads the AUTHORITATIVE REGISTRIES, not a list kept here: +// the host registry (src/lib/adapters), the managed-tool catalog the System area measures +// (src/lib/footprint/install.mjs), and the maintainer contract's own tools table +// (docs/MANAGED-TOOLS.md). Three independent authorities, all of which a new tool must pass +// through, so no single omission can let it ship uncarded. The reverse direction is checked +// against ak's own source: a card for a package ak never installs, or a "configured surface" +// naming a command ak does not ship, is a lie the same gate fails on. +// +// The register-contract tests cover only the MECHANICAL half of the editorial rules +// (docs/ddd/component-directory.md). Tone, accuracy, and plain language stay a review duty — +// length, superlatives, and the chip-vocabulary boundary do not have to be. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + CATEGORY_ORDER, directoryEntries, entryById, entriesByCategory, +} from '../../src/lib/dashboard/about-directory.mjs'; +import { HOST_REGISTRY } from '../../src/lib/adapters/index.mjs'; +import { managedTools } from '../../src/lib/footprint/install.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const MODULE_PATH = path.join(ROOT, 'src/lib/dashboard/about-directory.mjs'); +const MODULE_URL = new URL('../../src/lib/dashboard/about-directory.mjs', import.meta.url).href; + +const ENTRIES = directoryEntries(); +// The discriminator the directory itself uses: a configured surface is not a package, so it +// carries the status row its chip joins on and the command that changes it, never a version. +const isConfigured = (entry) => entry.category === 'configured'; +const packagedEntries = () => ENTRIES.filter((entry) => !isConfigured(entry)); + +const words = (text) => String(text).split(/\s+/).filter((word) => /[A-Za-z0-9]/.test(word)); + +/** Every identity a card may legitimately be found by. A managed tool is named differently + * by each authority — the host registry knows `claude`, npm knows `@anthropic-ai/claude-code`, + * the directory calls the card `claude-code` — so resolution accepts any of the three rather + * than forcing one authority's spelling onto the editorial ids. */ +function identityIndex() { + const index = new Map(); + for (const entry of ENTRIES) { + for (const key of [entry.id, entry.npmPackage, entry.detectionKey].filter(Boolean)) { + if (!index.has(key)) index.set(key, new Set()); + index.get(key).add(entry); + } + } + return index; +} + +const resolveEntries = (index, key) => [...(index.get(key) ?? [])]; + +/** The tool names in the managed-tools contract's own table. Parsed rather than transcribed: + * a transcript would drift the moment the table gains a row, which is the drift this gate + * exists to catch. An empty parse fails the test — a restructured doc must be re-read, not + * silently believed. */ +function managedToolsDocRows() { + const doc = readFileSync(path.join(ROOT, 'docs/MANAGED-TOOLS.md'), 'utf8'); + const section = doc.split(/^## /m).find((part) => part.startsWith('The tools')); + assert.ok(section, 'docs/MANAGED-TOOLS.md must still have a "## The tools" section'); + return [...section.matchAll(/^\|\s*\*\*(.+?)\*\*/gm)].map((match) => match[1].trim()); +} + +/** Two rows in that table name something other than a package id: `hosts` is a family (it + * expands through the host registry, asserted separately) and `kit (self)` is ak reporting on + * itself under its status key. Every other row must resolve on its own name — an unrecognized + * row is a new managed tool and fails. */ +const DOC_ROW_ALIASES = Object.freeze({ hosts: null, 'kit (self)': 'self' }); + +/** Every .mjs/.cjs line ak ships, minus the directory itself — the evidence that a packaged + * card names a package ak's own code actually installs, pins, or detects. */ +function kitSource() { + const chunks = []; + const walk = (dir) => { + for (const dirent of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, dirent.name); + if (dirent.isDirectory()) walk(full); + else if (/\.(mjs|cjs)$/.test(dirent.name) && full !== MODULE_PATH) { + chunks.push(readFileSync(full, 'utf8')); + } + } + }; + walk(path.join(ROOT, 'src')); + return chunks.join('\n'); +} + +/** The commands ak actually dispatches, read from the CLI's own dispatch tables so a renamed + * command surfaces here as a broken "yours to change" promise. */ +function shippedCommands() { + const bin = readFileSync(path.join(ROOT, 'bin/agentic-kit.mjs'), 'utf8'); + const table = (name) => { + const start = bin.indexOf(`const ${name} = `); + assert.notEqual(start, -1, `bin/agentic-kit.mjs must still declare ${name}`); + const block = bin.slice(start, bin.indexOf('});', start)); + const keys = new Set([...block.matchAll(/^\s*'?([\w-]+)'?:\s*\(\)\s*=>/gm)].map((m) => m[1])); + assert.ok(keys.size > 0, `${name} must parse to at least one command`); + return keys; + }; + return { porcelain: table('PORCELAIN'), plumbing: table('PLUMBING') }; +} + +// --------------------------------------------------------------------------------------- +// The parity gate (ADR-0026 §3, component-directory invariant 4) +// --------------------------------------------------------------------------------------- + +test('every managed tool has exactly one directory entry (registry → directory)', () => { + const index = identityIndex(); + const missing = []; + const duplicated = []; + + // Hosts resolve on their detection key, not their npm name, because the chip joins on + // `hosts.` — checking the key the renderer uses is what proves the card can be filled. + for (const host of HOST_REGISTRY) { + const matched = ENTRIES.filter((entry) => entry.detectionKey === `hosts.${host.id}`); + if (!matched.length) missing.push(`host '${host.id}' (${host.label})`); + if (matched.length > 1) duplicated.push(`host '${host.id}' → ${matched.map((e) => e.id)}`); + if (matched.length === 1) { + assert.equal(matched[0].npmPackage, host.install.npmPackage, + `${matched[0].id} must name the package the host registry installs`); + } + } + + // The catalog the System area measures: anything with bytes on disk is a thing the user + // can see occupying their machine, so it is a thing About owes an explanation for. + for (const tool of managedTools()) { + const key = tool.pkg ?? tool.id; + const matched = resolveEntries(index, key); + if (!matched.length) missing.push(`managed tool '${tool.id}' (${key})`); + if (matched.length > 1) duplicated.push(`managed tool '${key}' → ${matched.map((e) => e.id)}`); + } + + assert.deepEqual(missing, [], + 'these managed tools have no About card — add a directory entry before shipping them'); + assert.deepEqual(duplicated, [], 'a managed tool must map to exactly ONE card'); +}); + +test('the managed-tools contract names no tool the directory omits', () => { + const index = identityIndex(); + const rows = managedToolsDocRows(); + assert.ok(rows.length >= 5, `docs/MANAGED-TOOLS.md tools table parsed to ${rows.length} rows`); + + const missing = []; + for (const row of rows) { + if (row in DOC_ROW_ALIASES && DOC_ROW_ALIASES[row] === null) continue; // asserted above + const key = DOC_ROW_ALIASES[row] ?? row; + if (!resolveEntries(index, key).length) missing.push(`${row} (looked up as '${key}')`); + } + assert.deepEqual(missing, [], + 'MANAGED-TOOLS.md documents these tools but About introduces none of them'); + + // The family row is the one that cannot resolve by name; prove it is covered by ids. + assert.equal( + ENTRIES.filter((entry) => entry.detectionKey?.startsWith('hosts.')).length, + HOST_REGISTRY.length, + 'one host card per registered host, no more', + ); +}); + +test('every configured surface ADR-0026 promises has exactly one card', () => { + // Configured surfaces have no package registry to be measured against — the ADR's own + // layout is their authority, so it is what gets parsed. Both directions again: a surface + // ak sets up but never explains is the same failure as a card for a surface ak never + // touches, and deleting a card must not be able to pass quietly. + const adr = readFileSync(path.join(ROOT, 'docs/adr/0026-about-component-directory.md'), 'utf8'); + const listed = adr.match(/Non-package surfaces:([\s\S]*?)—/); + assert.ok(listed, 'ADR-0026 must still enumerate the non-package surfaces'); + // Singular/plural is a copy choice ("permission allowlists" / "Permission allowlist"), so + // the comparison is on identity, not on the exact words the two documents chose. + const norm = (name) => name.toLowerCase().replace(/[^a-z0-9]/g, '').replace(/s$/, ''); + const promised = listed[1].replace(/\s+/g, ' ').split(',').map((name) => name.trim()) + .filter(Boolean).map(norm); + assert.ok(promised.length >= 5, `ADR-0026 parsed to ${promised.length} configured surfaces`); + + const carded = ENTRIES.filter(isConfigured).map((entry) => norm(entry.name)); + assert.deepEqual([...carded].sort(), [...promised].sort(), + 'the "Configured for you" cards and the surfaces ADR-0026 promises must be the same set'); +}); + +test('no directory entry exists for something ak neither installs nor configures', () => { + const source = kitSource(); + const { porcelain, plumbing } = shippedCommands(); + const unjustified = []; + + for (const entry of ENTRIES) { + if (isConfigured(entry)) { + // A configured surface's claim is "ak set this up and you can change it" — the second + // half is falsifiable, so it is what gets checked. + const [ak, first, second] = String(entry.manage).split(/\s+/); + assert.equal(ak, 'ak', `${entry.id}: manage command must be an ak command`); + const known = first === 'x' ? plumbing.has(second) : porcelain.has(first); + if (!known) unjustified.push(`${entry.id}: '${entry.manage}' is not a command ak ships`); + continue; + } + // A packaged card claims ak installs, pins, or detects that package. ak's own source is + // the only place that could be true, so the package name has to appear in it — as a + // literal, not as a substring of some unrelated path. + const quoted = ['\'', '"', '`'].some((q) => source.includes(`${q}${entry.npmPackage}${q}`)); + if (!quoted) { + unjustified.push(`${entry.id}: ak's source never names the package '${entry.npmPackage}'`); + } + } + + assert.deepEqual(unjustified, [], + 'About must not introduce components ak does not install or configure'); +}); + +test('each entry is a package ak manages or a surface ak configures, never both', () => { + const ids = ENTRIES.map((entry) => entry.id); + assert.equal(new Set(ids).size, ids.length, 'directory ids must be unique'); + + for (const entry of ENTRIES) { + assert.ok(CATEGORY_ORDER.includes(entry.category), `${entry.id}: unknown category`); + assert.match(entry.id, /^[a-z0-9]+(-[a-z0-9]+)*$/, `${entry.id}: id must be kebab-case`); + assert.ok(entry.name?.length, `${entry.id}: name is required`); + if (isConfigured(entry)) { + assert.ok(entry.subsystem?.length, `${entry.id}: configured surface needs a status row key`); + assert.ok(entry.manage?.length, `${entry.id}: configured surface needs a manage command`); + assert.equal(entry.detectionKey, undefined, `${entry.id}: a surface is not a package`); + assert.equal(entry.npmPackage, undefined, `${entry.id}: a surface is not a package`); + } else { + assert.ok(entry.detectionKey?.length, `${entry.id}: packaged entry needs a detection key`); + assert.ok(entry.npmPackage?.length, `${entry.id}: packaged entry needs its package name`); + assert.equal(entry.subsystem, undefined, `${entry.id}: a package is not a configured row`); + } + } + + for (const field of ['detectionKey', 'npmPackage']) { + const seen = packagedEntries().map((entry) => entry[field]); + assert.equal(new Set(seen).size, seen.length, `two cards share a ${field} — one tool, one card`); + } + const subsystems = ENTRIES.filter(isConfigured).map((entry) => entry.subsystem); + assert.equal(new Set(subsystems).size, subsystems.length, + 'two surfaces share a status row — one surface, one card'); +}); + +// --------------------------------------------------------------------------------------- +// The register contract (docs/ddd/component-directory.md), mechanical half +// --------------------------------------------------------------------------------------- + +test('taglines stay inside their ten-word budget and read as one line', () => { + for (const entry of ENTRIES) { + const count = words(entry.tagline).length; + assert.ok(count <= 10, `${entry.id}: tagline is ${count} words (max 10) — "${entry.tagline}"`); + assert.ok(count >= 3, `${entry.id}: tagline is too short to say anything`); + assert.doesNotMatch(entry.tagline, /\n/, `${entry.id}: tagline must be one line`); + assert.match(entry.tagline, /\.$/, `${entry.id}: tagline is a sentence and ends like one`); + } +}); + +test('each paragraph is ONE paragraph inside the ~50-word band', () => { + for (const entry of ENTRIES) { + // The card's height is designed around this band (docs/assets/about-tab-mock.html); a + // paragraph outside it either says nothing or turns the card back into documentation. + const count = words(entry.paragraph).length; + assert.ok(count >= 40 && count <= 58, + `${entry.id}: paragraph is ${count} words, outside the 40–58 band around ~50`); + assert.doesNotMatch(entry.paragraph, /\n/, `${entry.id}: paragraph must be a single paragraph`); + assert.doesNotMatch(entry.paragraph, / {2,}/, `${entry.id}: collapsed whitespace only`); + assert.match(entry.paragraph, /^["'A-Z]/, `${entry.id}: paragraph starts a sentence`); + assert.match(entry.paragraph, /[.?]["']?$/, `${entry.id}: paragraph ends a sentence`); + } +}); + +test('no marketing superlatives anywhere in the authored copy', () => { + // Claims detection cannot back. The card's job is to say what the thing does for the user; + // "blazing" is not a fact the dashboard could ever render a chip for. Exclamation marks are + // banned by the same rule — warmth comes from clarity, not punctuation. + const superlatives = new RegExp([ + 'best', 'best-in-class', 'fastest', 'blazing(?:ly)?', 'world-class', 'industry-leading', + 'cutting-edge', 'state-of-the-art', 'next-generation', 'revolutionary', 'seamless(?:ly)?', + 'effortless(?:ly)?', 'powerful', 'unparalleled', 'unmatched', 'magical', 'amazing', + 'incredible', 'awesome', 'ultimate', 'supercharges?d?', 'game-changing', 'lightning-fast', + 'enterprise-grade', 'military-grade', 'robust', 'simply', 'just works', '10x', + ].map((word) => `\\b${word}\\b`).join('|'), 'i'); + + for (const entry of ENTRIES) { + for (const [field, text] of [['tagline', entry.tagline], ['paragraph', entry.paragraph]]) { + const hit = text.match(superlatives); + assert.equal(hit, null, `${entry.id}.${field}: superlative "${hit?.[0]}" — say what it does`); + assert.doesNotMatch(text, /!/, `${entry.id}.${field}: no exclamation marks`); + } + } +}); + +test('prose never claims runtime state — installed/running/healthy belong to the chip', () => { + // The chip vocabulary, banned outright from the tagline (too short for any other reading) + // and banned from the paragraph wherever it is PREDICATED of the component: "is installed", + // "already running", "stays healthy". A causative sentence about what ak does for a + // component ("keeps it healthy") is a statement of purpose that reads true on a machine + // where the component is absent, which is the invariant's actual test (component-directory + // invariant 3) — the ban exists so a card can never contradict its own honest chip. + const chipWords = /\b(installed|running|healthy)\b/i; + const predicated = new RegExp( + '\\b(?:is|are|was|were|be|been|being|stays?|remains?|already|currently|now|not)\\s+' + + '(?:\\w+\\s+){0,2}?(installed|running|healthy)\\b', 'i', + ); + + for (const entry of ENTRIES) { + const inTagline = entry.tagline.match(chipWords); + assert.equal(inTagline, null, + `${entry.id}.tagline: "${inTagline?.[0]}" is a chip word, not editorial copy`); + const claim = entry.paragraph.match(predicated); + assert.equal(claim, null, + `${entry.id}.paragraph: "${claim?.[0]}" claims runtime state the chip owns`); + assert.doesNotMatch(entry.paragraph, /\bnot installed\b|\bup and running\b/i, + `${entry.id}.paragraph: state phrasing belongs to the chip`); + } +}); + +// --------------------------------------------------------------------------------------- +// Links, order, icons (invariants 5, 9, 10) +// --------------------------------------------------------------------------------------- + +test('every link is https and carries kind, label, and url', () => { + const kinds = new Set(['github', 'npm', 'docs']); + for (const entry of ENTRIES) { + assert.ok(Array.isArray(entry.links), `${entry.id}: links must be an array`); + const seen = new Set(); + for (const link of entry.links) { + const where = `${entry.id} link ${link.kind ?? '?'}`; + assert.ok(kinds.has(link.kind), `${where}: unknown link kind`); + assert.equal(seen.has(link.kind), false, `${where}: one pill per kind`); + seen.add(link.kind); + assert.ok(link.label?.length, `${where}: a pill needs a label`); + assert.match(link.url, /^https:\/\//, `${where}: links are https, user-initiated`); + const url = new URL(link.url); + assert.ok(url.hostname.includes('.'), `${where}: needs a named host`); + assert.equal(url.username, '', `${where}: no credentials in a shipped link`); + assert.equal(link.url.trim(), link.url, `${where}: no stray whitespace`); + } + } +}); + +test('an npm pill points at the very package its chip reports on', () => { + // The pill and the chip must be about the same artifact, or a user clicks through from + // "installed v3.34.0" to a different package's page and the card has lied by juxtaposition. + for (const entry of packagedEntries()) { + const npm = entry.links.find((link) => link.kind === 'npm'); + if (!npm) continue; + assert.equal(npm.url, `https://www.npmjs.com/package/${entry.npmPackage}`, + `${entry.id}: npm pill must resolve to '${entry.npmPackage}'`); + } +}); + +test('packaged cards link out; configured surfaces name a command instead', () => { + for (const entry of packagedEntries()) { + assert.ok(entry.links.length >= 1, `${entry.id}: a component card needs its receipts`); + for (const kind of ['github', 'docs']) { + assert.ok(entry.links.some((link) => link.kind === kind), + `${entry.id}: no ${kind} link — the card is an introduction, depth lives behind it`); + } + } + for (const entry of ENTRIES.filter(isConfigured)) { + // A configured surface has no upstream of its own; the honest substitute for a link pill + // is the command that changes it, which the parity gate already proved ak ships. + assert.deepEqual(entry.links, [], `${entry.id}: a configured surface has nothing to link to`); + assert.match(entry.manage, /^ak /, `${entry.id}: manage command names ak`); + } +}); + +test('curated order is stable and follows the documented category sequence', () => { + assert.deepEqual([...CATEGORY_ORDER], [ + 'hosts', 'engine-memory', 'quality', 'safety', 'knowledge', 'kit', 'configured', + ], 'the category sequence is a documented decision (ADR-0026 §1), not a preference'); + + const rank = (entry) => CATEGORY_ORDER.indexOf(entry.category); + for (let i = 1; i < ENTRIES.length; i += 1) { + assert.ok(rank(ENTRIES[i]) >= rank(ENTRIES[i - 1]), + `${ENTRIES[i].id} breaks the curated order — categories may not interleave`); + } + assert.equal(ENTRIES[0].category, 'hosts', 'the things a user already recognizes come first'); + assert.equal(ENTRIES.at(-1).category, 'configured', 'configured surfaces come last'); + assert.equal(ENTRIES.find((entry) => entry.category === 'kit').id, 'agentic-kit'); + + // Hosts read in registry order so About, Observability, and `ak status` list them alike. + assert.deepEqual( + entriesByCategory('hosts').map((entry) => entry.detectionKey), + HOST_REGISTRY.map((host) => `hosts.${host.id}`), + ); + + assert.deepEqual(entriesByCategory('nonsense'), [], 'an unknown category yields nothing'); + assert.equal(entryById('nope'), null, 'an unknown id is an absence, never a partial card'); + assert.equal(entryById('ruflo').name, 'ruflo'); +}); + +test('official marks are used only for the three hosts; everything else is a monogram', () => { + const hostIds = new Set(HOST_REGISTRY.map((host) => host.id)); + for (const entry of ENTRIES) { + assert.ok(entry.icon, `${entry.id}: every card has an icon`); + if (entry.icon.kind === 'official') { + assert.equal(entry.category, 'hosts', + `${entry.id}: only the hosts ship a genuinely official mark`); + assert.ok(hostIds.has(entry.icon.ref), + `${entry.id}: official ref must be a host id the dashboard already renders`); + } else { + assert.equal(entry.icon.kind, 'monogram', `${entry.id}: icons are official or monogram`); + assert.match(entry.icon.ref, /^[A-Za-z]{1,2}$/, `${entry.id}: a monogram is its initials`); + assert.match(entry.icon.hue, /^--[a-z-]+$/, `${entry.id}: hue names a CSS token`); + } + } + assert.equal(ENTRIES.filter((entry) => entry.icon.kind === 'official').length, hostIds.size, + 'no fabricated brand assets, and no host left without its real one'); +}); + +// --------------------------------------------------------------------------------------- +// Purity (invariants 1 and 2: authored, versioned, never generated or fetched) +// --------------------------------------------------------------------------------------- + +test('the directory is data: two module instances agree and nothing is rebuilt', async () => { + // A second, cache-busted instantiation of the same source. Equal data from an independent + // evaluation proves the catalog is not assembled from anything ambient. + const twice = await import(`${MODULE_URL}?instance=2`); + assert.deepEqual(twice.directoryEntries(), ENTRIES); + assert.notStrictEqual(twice.directoryEntries(), ENTRIES, 'the two instances must be distinct'); + assert.strictEqual(directoryEntries(), ENTRIES, 'accessors return the frozen catalog itself'); + + assert.ok(Object.isFrozen(ENTRIES), 'the catalog is frozen'); + for (const entry of ENTRIES) { + assert.ok(Object.isFrozen(entry), `${entry.id}: entry must be frozen`); + assert.ok(Object.isFrozen(entry.links), `${entry.id}: links must be frozen`); + assert.ok(Object.isFrozen(entry.icon), `${entry.id}: icon must be frozen`); + for (const link of entry.links) assert.ok(Object.isFrozen(link), `${entry.id}: link frozen`); + } +}); + +test('the directory module reads nothing — no I/O surface exists in it at all', () => { + // Editorial content is authored and versioned WITH THE RELEASE (invariant 1) and the + // directory collects nothing of its own (invariant 2). Both hold structurally as long as + // this module has no way to reach a filesystem, a network, a clock, or an environment — + // so the gate is on the source, where the capability would have to appear first. + const source = readFileSync(MODULE_PATH, 'utf8'); + const forbidden = [ + [/from\s+'node:/, 'imports a node builtin'], + [/\brequire\s*\(/, 'uses require()'], + [/\bimport\s*\(/, 'imports dynamically'], + [/\bprocess\.[a-z]/i, 'reads process state'], + [/\bfetch\s*\(/, 'fetches'], + [/new Date\b|Date\.now\b/, 'reads the clock'], + [/Math\.random\b/, 'is nondeterministic'], + ]; + for (const [pattern, why] of forbidden) { + assert.doesNotMatch(source, pattern, `about-directory.mjs ${why} — it must stay pure data`); + } +}); From ede6159abe8afa8be73ca034683ea0ea0613600f Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 16:31:39 -0700 Subject: [PATCH 06/19] feat(system): machine-footprint collectors incl. first-class Windows census MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements ADR-0025's collectors: a bounded walker (never follows symlinks, one bad subtree degrades to unknown), install/storage/catalog/projects, and an ephemeral runtime census. Unknown is never rendered as zero — every figure carries measured/partial/unknown provenance, and lower bounds print as ">= N". Windows is real rather than unsupported. A shipped PowerShell script gives the guaranteed floor via Get-CimInstance (pid/ppid/CPU/RSS/uptime, argv deliberately excluded), and a best-effort inline P/Invoke walks NtQueryInformationProcess -> PEB -> RTL_USER_PROCESS_PARAMETERS for true cwd. A bitness mismatch is detected rather than read through with wrong offsets, and any probe failure degrades to an honest "not attributable" — never a fabricated path, and never taking the census down with it. No dependency added; package.json still declares none. The script lives beside its consumer under src/ because `files` ships src/ wholesale — under scripts/ it would never have reached an npm-installed Windows user at all. A guard test asserts that placement on every platform, and three live tests execute the real PowerShell on windows-latest, checking the census and the PEB walk against this process's own pid and cwd. --- src/lib/footprint/catalog.mjs | 497 +++++++++++ src/lib/footprint/index.mjs | 389 ++++++++ src/lib/footprint/install.mjs | 467 ++++++++++ src/lib/footprint/projects.mjs | 457 ++++++++++ src/lib/footprint/runtime.mjs | 204 +++++ src/lib/footprint/snapshot.mjs | 250 ++++++ src/lib/footprint/storage.mjs | 663 ++++++++++++++ src/lib/footprint/walk.mjs | 289 ++++++ src/lib/live/process-sessions.mjs | 452 ++++++++-- src/lib/live/win-process-survey.ps1 | 249 ++++++ tests/kit/footprint-collectors.test.mjs | 989 +++++++++++++++++++++ tests/kit/footprint-windows.test.mjs | 428 +++++++++ tests/kit/win-process-survey-live.test.mjs | 124 +++ 13 files changed, 5407 insertions(+), 51 deletions(-) create mode 100644 src/lib/footprint/catalog.mjs create mode 100644 src/lib/footprint/index.mjs create mode 100644 src/lib/footprint/install.mjs create mode 100644 src/lib/footprint/projects.mjs create mode 100644 src/lib/footprint/runtime.mjs create mode 100644 src/lib/footprint/snapshot.mjs create mode 100644 src/lib/footprint/storage.mjs create mode 100644 src/lib/footprint/walk.mjs create mode 100644 src/lib/live/win-process-survey.ps1 create mode 100644 tests/kit/footprint-collectors.test.mjs create mode 100644 tests/kit/footprint-windows.test.mjs create mode 100644 tests/kit/win-process-survey-live.test.mjs diff --git a/src/lib/footprint/catalog.mjs b/src/lib/footprint/catalog.mjs new file mode 100644 index 0000000..4edaeda --- /dev/null +++ b/src/lib/footprint/catalog.mjs @@ -0,0 +1,497 @@ +// Catalog inventory — the deduplicated set of skills, agents, commands, plugins, +// and MCP servers actually deployed across hosts, each with a per-host presence +// matrix (which host carries it, observed from which surface). +// +// OBSERVED inventory only (ADR-0025 invariant 10): this module reports what is on +// disk per host surface and never reads, mirrors, or upgrades Integration +// management's desired state. +// +// Counting is by NAME — directory entries, manifest keys, TOML table names. Item +// bodies are never parsed (invariant 1). Directory surfaces go through the shared +// bounded walker, which is why nothing here follows a symlink or escapes a root; +// the manifest surfaces are single-file reads of keys, the same spawn-free seam +// mcp.mjs already uses because `claude mcp list` health-checks every server and +// has no stable schema. +// +// The single content read is the managed-block sentinel scan on guidance files, +// which ADR-0025's config-surface row explicitly sanctions; it retains a count +// and nothing else. +import fs from 'node:fs'; +import path from 'node:path'; +import { + claudeDir, claudeSettingsPath, claudeUserMcpPath, + codexDir, codexConfigPath, + opencodeDir, opencodeConfigPath, + projectSettings, projectSettingsLocal, repoRoot, +} from '../paths.mjs'; +import { readJson } from '../settings.mjs'; +import { inspectCodexPlugins } from '../codex-plugins.mjs'; +import { registry, blocksForTarget, guidanceTargets, hasBlock } from '../blocks.mjs'; +import { walkTree, statNode, measured, unknown } from './walk.mjs'; + +/** Per-surface name cap. A capped surface reports truncated — its count is then a + * floor, never a total. */ +const MAX_NAMES = 4096; + +export const CATALOG_KINDS = ['skill', 'agent', 'command', 'plugin', 'mcpServer']; +export const CATALOG_HOSTS = ['claude', 'codex', 'opencode']; + +/** Any BEGIN sentinel, kit-managed or not — the count of foreign or orphaned + * blocks is the interesting half of "what is deployed in my guidance files". */ +const SENTINEL_RE = /^$/gm; + +// ── surface readings ────────────────────────────────────────────────────────── + +/** + * One surface's names plus how the read went, in the same three-valued vocabulary + * usage-index's rootHealth uses: 'ok' (read it), 'absent' (nothing there — a real + * zero), 'degraded' (could not look — unknown, never zero). `partial` marks a + * count that is a floor because a cap fired or a subtree was unreadable. + * @typedef {{ status: 'ok'|'absent'|'degraded', reason: string|null, + * names: string[], partial: boolean, truncated: boolean }} SurfaceReading + */ + +const emptyReading = (status, reason) => ({ status, reason, names: [], partial: false, truncated: false }); + +/** + * Names below `root`, one per file the walker accepts. `nameOf` turns the accepted + * file's path into the catalog name — the ':'-joined relative path a host uses to + * namespace nested entries. + * + * Scope is enforced with `skipDir` rather than `maxDepth` on purpose: a catalog + * surface is a shallow name space, and everything below it is an item's own + * reference material, not another item. Pruning it is a deliberate scope (the + * walker does not call that truncation), whereas a depth cap would mark every + * skill that ships a `references/` folder as an incomplete measurement. + */ +function readNames(root, { accept, nameOf, dirDepth, walk = walkTree, limits = {}, fsImpl = fs }) { + const names = []; + const result = walk(root, { + ...limits, fsImpl, + skipDir: (dir, name, depth) => depth > dirDepth, + acceptFile: (name) => accept(name), + onFile: ({ file }) => { + if (names.length >= MAX_NAMES) return; + const name = nameOf(file); + if (name) names.push(name); + }, + }); + if (result.status === 'unknown') { + return emptyReading(result.reason === 'ENOENT' ? 'absent' : 'degraded', result.reason); + } + const truncated = Boolean(result.truncated) || names.length >= MAX_NAMES; + return { + status: 'ok', + reason: result.degraded?.[0]?.reason ?? null, + names, + partial: result.complete === false || truncated, + truncated, + }; +} + +/** ':'-joined path of `file` relative to `root`, minus a trailing extension. */ +function relativeName(root, file, { strip = '' } = {}) { + let rel = path.relative(root, file); + if (!rel || rel.startsWith('..')) return null; + if (strip && rel.endsWith(strip)) rel = rel.slice(0, -strip.length); + return rel.split(path.sep).filter(Boolean).join(':'); +} + +/** Directories carrying `marker` (SKILL.md), named by their path below `root`. + * Two directory levels: a bare `/` and a namespaced `//`. */ +const readMarkerDirs = (root, marker, opts = {}) => readNames(root, { + dirDepth: 2, + accept: (name) => name === marker, + nameOf: (file) => relativeName(root, path.dirname(file)), + ...opts, +}); + +/** `*.md` entries below `root`. A README documents the surface rather than being + * an entry on it, so it is excluded instead of counted as a command. */ +const readMarkdownNames = (root, opts = {}) => readNames(root, { + dirDepth: 3, + accept: (name) => name.endsWith('.md') && !/^readme\.md$/i.test(name), + nameOf: (file) => relativeName(root, file, { strip: '.md' }), + ...opts, +}); + +/** Top-level files with one of `exts`, named by basename without the extension. */ +const readFileStems = (root, exts, opts = {}) => readNames(root, { + dirDepth: 0, + accept: (name) => exts.includes(path.extname(name)), + nameOf: (file) => path.basename(file, path.extname(file)), + ...opts, +}); + +/** Keys of one object inside a JSON manifest. */ +function readManifestKeys(file, pick, { fsImpl = fs } = {}) { + const head = statNode(file, { fsImpl }); + if (head.status === 'unknown') { + return emptyReading(head.reason === 'ENOENT' ? 'absent' : 'degraded', head.reason); + } + const doc = readJson(file, null); + if (doc === null) return emptyReading('degraded', 'EPARSE'); + const bag = pick(doc); + if (!bag || typeof bag !== 'object') return { ...emptyReading('ok', null), names: [] }; + return { status: 'ok', reason: null, names: Object.keys(bag), partial: false, truncated: false }; +} + +/** TOML table names under `section` — the same regex-over-source approach + * codex-plugins' enabledPluginRefs uses, because codex owns config.toml and this + * kit only ever observes it. Exported for tests. */ +export function tomlTableNames(source, section) { + const names = []; + const re = new RegExp( + `^\\[\\s*${section}\\s*\\.\\s*(?:"((?:[^"\\\\]|\\\\.)+)"|'([^']+)'|([A-Za-z0-9_.\\-]+))\\s*\\]\\s*$`, + 'gm', + ); + let match; + while ((match = re.exec(source)) !== null) { + const quoted = match[1]; + names.push(quoted ? quoted.replace(/\\"/g, '"').replace(/\\\\/g, '\\') : (match[2] ?? match[3])); + } + return names; +} + +function readTomlTables(file, section, { fsImpl = fs } = {}) { + let source; + try { source = fsImpl.readFileSync(file, 'utf8'); } catch (error) { + return emptyReading(error.code === 'ENOENT' ? 'absent' : 'degraded', error.code ?? 'io'); + } + return { status: 'ok', reason: null, names: tomlTableNames(source, section), partial: false, truncated: false }; +} + +/** Claude's installed-plugin manifest: `{ plugins: { "@": [ … ] } }`. + * Returns the plugin ids plus the newest install root per id — the roots the + * plugin-provided skill/agent/command surfaces hang off. */ +function readClaudePlugins(file, { fsImpl = fs } = {}) { + const head = statNode(file, { fsImpl }); + if (head.status === 'unknown') { + return { ...emptyReading(head.reason === 'ENOENT' ? 'absent' : 'degraded', head.reason), roots: [] }; + } + const doc = readJson(file, null); + if (doc === null) return { ...emptyReading('degraded', 'EPARSE'), roots: [] }; + const names = []; + const roots = []; + for (const [ref, installs] of Object.entries(doc?.plugins ?? {})) { + const id = ref.includes('@') ? ref.slice(0, ref.lastIndexOf('@')) : ref; + names.push(id); + const newest = (Array.isArray(installs) ? installs : []).at(-1); + if (newest?.installPath) roots.push({ id, root: newest.installPath }); + } + return { status: 'ok', reason: null, names, partial: false, truncated: false, roots }; +} + +// ── surface specs ───────────────────────────────────────────────────────────── + +/** + * Every catalog surface this machine could carry, as data. A surface that does + * not exist reads 'absent' and contributes a real zero — which is why a host's + * documented-but-unused convention (codex prompts, opencode commands) is safe to + * list: it costs one stat and can never fabricate an entry. + */ +function surfaceSpecs(roots, io) { + const { claudeRoot, claudeMcpFile, codexRoot, codexConfigFile, opencodeRoot, opencodeConfigFile, cwd } = roots; + const at = (base, ...rest) => path.join(base, ...rest); + const specs = []; + + // claude — user scope + specs.push({ id: 'claude-skills', host: 'claude', kind: 'skill', path: at(claudeRoot, 'skills'), + read: (p) => readMarkerDirs(p, 'SKILL.md', io) }); + specs.push({ id: 'claude-agents', host: 'claude', kind: 'agent', path: at(claudeRoot, 'agents'), + read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: 'claude-commands', host: 'claude', kind: 'command', path: at(claudeRoot, 'commands'), + read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: 'claude-plugins', host: 'claude', kind: 'plugin', + path: at(claudeRoot, 'plugins', 'installed_plugins.json'), + read: (p) => readClaudePlugins(p, io) }); + specs.push({ id: 'claude-user-mcp', host: 'claude', kind: 'mcpServer', path: claudeMcpFile, + read: (p) => readManifestKeys(p, (d) => d?.mcpServers, io) }); + + // claude — project scope. The repo root is the same seam mcp.mjs's codexMcpStatus + // reads; outside a repo there is simply no project surface to report. + const projectRoot = repoRoot(cwd); + if (projectRoot) { + specs.push({ id: 'claude-project-mcp', host: 'claude', kind: 'mcpServer', + path: at(projectRoot, '.mcp.json'), read: (p) => readManifestKeys(p, (d) => d?.mcpServers, io) }); + } + + // codex + specs.push({ id: 'codex-skills', host: 'codex', kind: 'skill', path: at(codexRoot, 'skills'), + read: (p) => readMarkerDirs(p, 'SKILL.md', io) }); + specs.push({ id: 'codex-prompts', host: 'codex', kind: 'command', path: at(codexRoot, 'prompts'), + read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: 'codex-mcp', host: 'codex', kind: 'mcpServer', path: codexConfigFile, + read: (p) => readTomlTables(p, 'mcp_servers', io) }); + + // opencode + specs.push({ id: 'opencode-agents', host: 'opencode', kind: 'agent', path: at(opencodeRoot, 'agents'), + read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: 'opencode-skills', host: 'opencode', kind: 'skill', path: at(opencodeRoot, 'skills'), + read: (p) => readMarkerDirs(p, 'SKILL.md', io) }); + specs.push({ id: 'opencode-commands', host: 'opencode', kind: 'command', path: at(opencodeRoot, 'command'), + read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: 'opencode-plugins', host: 'opencode', kind: 'plugin', path: at(opencodeRoot, 'plugins'), + read: (p) => readFileStems(p, ['.js', '.mjs', '.cjs', '.ts'], io) }); + specs.push({ id: 'opencode-mcp', host: 'opencode', kind: 'mcpServer', path: opencodeConfigFile, + read: (p) => readManifestKeys(p, (d) => d?.mcp, io) }); + + return specs; +} + +/** + * Sub-surfaces contributed by an installed plugin's own cache directory. Names are + * namespaced `:` — the convention the hosts themselves use, and + * what keeps a plugin's `migrate` command distinct from a user's. + * + * Two layouts are in the wild and both are read: content at the plugin root + * (`skills/`) and content under a nested `.claude/` (the same shape opencode's + * catalogSource probes for on a ruflo checkout). A plugin using one layout reports + * the other absent; an item found in both dedupes to one row with two presence + * entries. + */ +function pluginSubSurfaces(host, pluginId, root, idPrefix, io) { + const specs = []; + for (const [tag, base] of [['', root], ['dot', path.join(root, '.claude')]]) { + const id = (kind) => `${idPrefix}:${pluginId}:${tag ? `${tag}-` : ''}${kind}`; + specs.push({ id: id('skills'), host, kind: 'skill', path: path.join(base, 'skills'), + prefix: pluginId, read: (p) => readMarkerDirs(p, 'SKILL.md', io) }); + specs.push({ id: id('agents'), host, kind: 'agent', path: path.join(base, 'agents'), + prefix: pluginId, read: (p) => readMarkdownNames(p, io) }); + specs.push({ id: id('commands'), host, kind: 'command', path: path.join(base, 'commands'), + prefix: pluginId, read: (p) => readMarkdownNames(p, io) }); + } + // A plugin may ship its own MCP servers; those are registered under the + // plugin's identity, not the user's, so they carry the plugin namespace too. + specs.push({ id: `${idPrefix}:${pluginId}:mcp`, host, kind: 'mcpServer', + path: path.join(root, '.mcp.json'), prefix: pluginId, + read: (p) => readManifestKeys(p, (d) => d?.mcpServers, io) }); + return specs; +} + +// ── config surface ──────────────────────────────────────────────────────────── + +/** Size of a known file. A missing file is a measured zero flagged + * `present: false`, so a renderer says "not present" rather than printing 0 B. */ +function fileSize(file, { asOf = null, fsImpl = fs } = {}) { + const node = statNode(file, { fsImpl }); + if (node.status === 'unknown') { + return node.reason === 'ENOENT' + ? { ...measured(0, { asOf }), present: false, mtimeMs: null, path: file } + : { ...unknown(node.reason), present: null, mtimeMs: null, path: file }; + } + return { ...measured(node.bytes ?? 0, { asOf }), present: true, mtimeMs: node.mtimeMs, path: file }; +} + +/** + * The config-surface row: how many managed blocks each guidance file carries and + * how big the settings files are. Guidance files are the one place this domain + * reads content, and it reads only sentinel LINES — `hasBlock` answers "is this + * registry slug present" and the sentinel regex counts BEGIN markers. No prose, + * no block bodies, nothing retained past the count. + * + * @param {{ cwd?: string, cfg?: { customBlocks?: object[] }, asOf?: number|null, + * extraSettingsFiles?: Array<{ id: string, label: string, path: string }>, + * fsImpl?: typeof fs }} [options] + */ +export function collectConfigSurface({ + cwd = process.cwd(), + cfg = /** @type {{ customBlocks?: object[] }} */ ({}), + asOf = null, + extraSettingsFiles = [], + fsImpl = fs, +} = {}) { + const rows = registry(cfg?.customBlocks ?? []); + const guidance = []; + for (const target of guidanceTargets({ cwd })) { + const bytes = fileSize(target.file, { asOf, fsImpl }); + const expected = blocksForTarget(rows, target.name); + let content; + try { content = fsImpl.readFileSync(target.file, 'utf8'); } catch (error) { + const absent = error.code === 'ENOENT'; + guidance.push({ + name: target.name, label: target.label, path: target.file, bytes, + managed: absent ? measured(0, { asOf }) : unknown(error.code ?? 'io'), + observed: absent ? measured(0, { asOf }) : unknown(error.code ?? 'io'), + expected: expected.length, + slugs: [], + }); + continue; + } + const present = expected.filter((row) => hasBlock(content, row.slug)); + const observed = (content.match(SENTINEL_RE) ?? []).length; + guidance.push({ + name: target.name, + label: target.label, + path: target.file, + bytes, + managed: measured(present.length, { asOf }), + observed: measured(observed, { asOf }), + expected: expected.length, + slugs: present.map((row) => row.slug), + }); + // `content` dies with this iteration: the counts above are the whole payload, + // and no prose from a guidance file leaves this scope. + } + + const projectRoot = repoRoot(cwd); + const settings = [ + { id: 'claude-settings', label: '~/.claude/settings.json', file: claudeSettingsPath() }, + { id: 'claude-user-mcp', label: '~/.claude.json', file: claudeUserMcpPath() }, + { id: 'codex-config', label: '~/.codex/config.toml', file: codexConfigPath() }, + { id: 'opencode-config', label: 'opencode.json', file: opencodeConfigPath() }, + ]; + if (projectRoot) { + settings.push({ id: 'project-settings', label: '.claude/settings.json', file: projectSettings(projectRoot) }); + settings.push({ id: 'project-settings-local', label: '.claude/settings.local.json', file: projectSettingsLocal(projectRoot) }); + } + for (const extra of extraSettingsFiles) settings.push({ id: extra.id, label: extra.label, file: extra.path }); + + return { + guidance, + settings: settings.map((row) => ({ id: row.id, label: row.label, ...fileSize(row.file, { asOf, fsImpl }) })), + }; +} + +// ── assembly ────────────────────────────────────────────────────────────────── + +/** Dedup key. Case and surrounding whitespace are presentation, not identity; + * kind is part of the key because a `tester` agent and a `tester` command are two + * different deployed things. */ +const itemKey = (kind, name) => `${kind}::${name.trim().toLowerCase()}`; + +/** + * Read every host catalog surface and fold it into deduplicated CatalogItems. + * + * A count is `partial` when any surface feeding it was unreadable or capped: the + * value is then a measured LOWER BOUND. Calling it complete would overstate the + * evidence; calling it unknown would throw away what was actually observed. + * + * @param {{ claudeRoot?: string, claudeMcpFile?: string, codexRoot?: string, + * codexConfigFile?: string, opencodeRoot?: string, opencodeConfigFile?: string, + * cwd?: string, cfg?: object, now?: () => number, walk?: Function, + * limits?: object, fsImpl?: typeof fs, + * inspectCodexPlugins?: Function, includePluginSurfaces?: boolean }} [options] + * @returns {object} CatalogInventory + */ +export function collectCatalog({ + claudeRoot = claudeDir(), + claudeMcpFile = claudeUserMcpPath(), + codexRoot = codexDir(), + codexConfigFile = codexConfigPath(), + opencodeRoot = opencodeDir(), + opencodeConfigFile = opencodeConfigPath(), + cwd = process.cwd(), + cfg = {}, + now = Date.now, + walk = walkTree, + limits = {}, + fsImpl = fs, + inspectCodexPlugins: inspectCodexPluginsImpl = inspectCodexPlugins, + includePluginSurfaces = true, +} = {}) { + const asOf = now(); + const io = { walk, limits, fsImpl }; + const roots = { claudeRoot, claudeMcpFile, codexRoot, codexConfigFile, opencodeRoot, opencodeConfigFile, cwd }; + const specs = surfaceSpecs(roots, io); + + // Plugin sub-surfaces are discovered from the plugin manifests themselves, so + // they are appended after the base specs rather than hardcoded. + if (includePluginSurfaces) { + const claudePlugins = readClaudePlugins(path.join(claudeRoot, 'plugins', 'installed_plugins.json'), io); + for (const { id, root } of claudePlugins.roots) { + specs.push(...pluginSubSurfaces('claude', id, root, 'claude-plugin', io)); + } + let codexPlugins = { configPresent: false, plugins: [] }; + // Codex owns config.toml and its plugin cache; an unreadable one is not this + // kit's failure to fix, and it must not take the rest of the catalog with it. + try { codexPlugins = inspectCodexPluginsImpl({ configFile: codexConfigFile }) ?? codexPlugins; } + catch { codexPlugins = { configPresent: false, plugins: [] }; } + for (const plugin of codexPlugins.plugins ?? []) { + if (!plugin?.root || !plugin?.ref) continue; + const id = plugin.ref.includes('@') ? plugin.ref.slice(0, plugin.ref.lastIndexOf('@')) : plugin.ref; + specs.push(...pluginSubSurfaces('codex', id, plugin.root, 'codex-plugin', io)); + } + // Codex's enabled refs ARE that host's plugin inventory. + const enabled = (codexPlugins.plugins ?? []).map((plugin) => plugin?.ref).filter(Boolean); + specs.push({ + id: 'codex-plugins', host: 'codex', kind: 'plugin', path: codexConfigFile, + read: () => (codexPlugins.configPresent === false + ? emptyReading('absent', 'ENOENT') + : { status: 'ok', reason: null, names: enabled, partial: false, truncated: false }), + }); + } + + const items = new Map(); + const surfaces = []; + for (const spec of specs) { + const reading = spec.read(spec.path); + surfaces.push({ + id: spec.id, + host: spec.host, + kind: spec.kind, + path: spec.path, + status: reading.status, + reason: reading.reason ?? null, + partial: Boolean(reading.partial), + truncated: Boolean(reading.truncated), + // A degraded surface has NO count: we did not look, so there is no number. + count: reading.status === 'degraded' ? null : reading.names.length, + }); + if (reading.status === 'degraded') continue; + for (const raw of reading.names) { + const name = spec.prefix ? `${spec.prefix}:${raw}` : raw; + const key = itemKey(spec.kind, name); + let item = items.get(key); + if (!item) { + item = { key, kind: spec.kind, name, hosts: [], presence: [] }; + items.set(key, item); + } + if (!item.hosts.includes(spec.host)) item.hosts.push(spec.host); + item.presence.push({ host: spec.host, surface: spec.id, path: spec.path }); + } + } + + const incomplete = new Set(); + const incompleteByHost = new Set(); + for (const surface of surfaces) { + if (surface.status !== 'degraded' && !surface.partial) continue; + incomplete.add(surface.kind); + incompleteByHost.add(`${surface.host}::${surface.kind}`); + } + + const list = [...items.values()].sort((a, b) => (a.kind === b.kind + ? a.name.localeCompare(b.name) + : CATALOG_KINDS.indexOf(a.kind) - CATALOG_KINDS.indexOf(b.kind))); + + const counts = {}; + const perHost = {}; + for (const kind of CATALOG_KINDS) { + const value = list.filter((item) => item.kind === kind).length; + counts[kind] = measured(value, { asOf, partial: incomplete.has(kind) }); + } + for (const host of CATALOG_HOSTS) { + perHost[host] = {}; + for (const kind of CATALOG_KINDS) { + const value = list.filter((item) => item.kind === kind && item.hosts.includes(host)).length; + perHost[host][kind] = measured(value, { asOf, partial: incompleteByHost.has(`${host}::${kind}`) }); + } + } + + const degraded = surfaces.filter((surface) => surface.status === 'degraded').map((surface) => surface.id); + const truncated = surfaces.filter((surface) => surface.truncated).map((surface) => surface.id); + + return { + asOf, + hosts: CATALOG_HOSTS, + kinds: CATALOG_KINDS, + items: list, + counts, + perHost, + surfaces, + config: collectConfigSurface({ cwd, cfg, asOf, fsImpl }), + complete: degraded.length === 0 && truncated.length === 0, + degraded, + truncated, + }; +} diff --git a/src/lib/footprint/index.mjs b/src/lib/footprint/index.mjs new file mode 100644 index 0000000..de8474f --- /dev/null +++ b/src/lib/footprint/index.mjs @@ -0,0 +1,389 @@ +// footprint/index.mjs — the composed machine-footprint collector (ADR-0025). +// +// One collector behind two surfaces (the dashboard's System area and the +// `ak system` CLI), in two tiers that differ by orders of magnitude in cost: +// +// CHEAP runtime census + individually-known file stats + the last persisted +// deep snapshot, carried forward with ITS asOf. Served on every read, +// TTL-cached ~60s in memory following buildProjectSnapshotCache's +// pattern in dashboard-server.mjs — machine-wide data, one entry per +// collector instance, no per-caller key. +// DEEP the full storage walk + per-project LOC + cross-host catalog dedup. +// Explicit, user-triggered, SINGLE-FLIGHT: a second request attaches +// to the running scan instead of racing it (invariant 7). usage-index +// keys its coalescing map by the options that change the RESULT; a +// deep scan has exactly one identity per collector instance, so the +// same discipline collapses to a single in-flight promise here. +// +// Honest degradation is structural, not decorative (ADR-0023, invariant 2): a +// section that has never been deep-scanned is `null` with a reason next to it, +// never an object full of zeros — there is no numeric field for a renderer to +// misread. A measured zero is a real zero and stays one. +// +// KNOWN COST, deliberately accepted for v1: the deep collectors are +// synchronous, so a scan occupies the event loop in multi-second stretches. +// The tier boundary is what contains this — nothing on the cheap path walks a +// tree — and the scan yields between phases so an embedding server is not +// blocked for the whole run. Moving the walk off-thread is a separate +// decision with its own seam. +import fs from 'node:fs'; +import path from 'node:path'; +import { + claudeDir, claudeSettingsPath, claudeUserMcpPath, codexConfigPath, codexDir, configDir, home, +} from '../paths.mjs'; +import { loadKitConfig } from '../config.mjs'; +import { discoverRuvfloProjects } from '../dashboard/project-discovery.mjs'; +import { defaultOpencodeDbPath } from '../usage-opencode.mjs'; +import { UNKNOWN, measured, statNode, unknown } from './walk.mjs'; +import { collectRuntimeCensus } from './runtime.mjs'; +import { collectInstall } from './install.mjs'; +import { collectStorage } from './storage.mjs'; +import { collectCatalog } from './catalog.mjs'; +import { collectProjects } from './projects.mjs'; +import { + SNAPSHOT_SECTIONS, SNAPSHOT_STALE_AFTER_MS, carryForward, readSnapshot, snapshotFreshness, + snapshotPath, summarizeCompleteness, writeSnapshot, +} from './snapshot.mjs'; + +/** Same window as the project-snapshot cache: long enough that a burst of + * polls costs one census, short enough that the Runtime view is not lying. */ +export const CHEAP_TTL_MS = 60_000; + +/** Deep-scan progress phases, in order. `idle` is the state before any scan + * has run in this process; `done`/`failed` are terminal. */ +export const SCAN_PHASES = Object.freeze([ + 'idle', 'install', 'storage', 'catalog', 'projects', 'persist', 'done', 'failed', +]); + +/** Individually-known files the cheap tier can stat without a walk. Each is one + * lstat, so the whole list is affordable on every request — which is the + * point: these are the files that grow fastest between deep scans (ledgers, + * tee files, index caches), and a user watching one grow should not have to + * run a deep scan to see it move. */ +function knownFileSpecs() { + const stateRoot = process.env.XDG_STATE_HOME || path.join(home, '.local', 'state'); + const kit = (name) => path.join(configDir(), name); + // [id, host, category, label, path] — the categories are STORAGE_CATEGORIES' + // vocabulary so a known file and its deep-tier node land in the same bucket. + const rows = [ + ['claude-history', 'claude', 'ledgers-and-logs', 'history.jsonl', path.join(claudeDir(), 'history.jsonl')], + ['claude-settings', 'claude', 'kit-caches', 'settings.json', claudeSettingsPath()], + ['claude-user-mcp', 'claude', 'kit-caches', '.claude.json', claudeUserMcpPath()], + ['codex-history', 'codex', 'ledgers-and-logs', 'history.jsonl', path.join(codexDir(), 'history.jsonl')], + ['codex-config', 'codex', 'kit-caches', 'config.toml', codexConfigPath()], + ['opencode-store', 'opencode', 'transcripts', 'opencode.db', defaultOpencodeDbPath()], + ['ak-usage-index', 'agentic-kit', 'kit-caches', 'usage-index.json', kit('usage-index.json')], + ['ak-observability-workspaces', 'agentic-kit', 'kit-caches', 'observability-workspaces.json', + kit('observability-workspaces.json')], + ['ak-kit-config', 'agentic-kit', 'kit-caches', 'kit.json', kit('kit.json')], + ['ak-claude-limits', 'agentic-kit', 'kit-caches', 'claude-rate-limits.json', + kit('claude-rate-limits.json')], + ['ak-codex-limits', 'agentic-kit', 'kit-caches', 'codex-rate-limits.json', + kit('codex-rate-limits.json')], + ['ak-footprint-snapshot', 'agentic-kit', 'kit-caches', 'footprint-snapshot.json', snapshotPath()], + ['ak-runtime-debug', 'agentic-kit', 'ledgers-and-logs', 'runtime-debug.log', + path.join(stateRoot, 'agentic-kit', 'runtime-debug.log')], + ]; + return rows.map(([id, host, category, label, at]) => ({ id, host, category, label, path: at })); +} + +/** + * Stat the known files. ENOENT is an ABSENCE (a measured zero, presence + * 'absent') because a file that does not exist genuinely holds no bytes; every + * other errno is unknown-with-reason, matching walk.mjs's rootMeasurements + * vocabulary so the two agree on what "we saw nothing" means. + * + * @param {{ asOf?: number, fsImpl?: typeof fs, + * specs?: Array<{ id: string, host: string, category: string, + * label: string, path: string }> }} [options] + */ +export function knownFileNodes({ asOf = Date.now(), fsImpl = fs, specs = null } = {}) { + return (specs ?? knownFileSpecs()).map((spec) => { + const stat = statNode(spec.path, { fsImpl }); + if (stat.status === UNKNOWN) { + const absent = stat.reason === 'ENOENT'; + return { + ...spec, + presence: absent ? 'absent' : 'degraded', + kind: null, + bytes: absent ? measured(0, { asOf }) : unknown(stat.reason), + mtimeMs: null, + }; + } + return { + ...spec, + presence: 'present', + kind: stat.kind, + // A directory or symlink where a file was expected has no size we may + // claim — statNode refuses to follow the link, so there is nothing to + // report but the fact that it is not a regular file. + bytes: stat.kind === 'file' + ? measured(stat.bytes, { asOf }) + : unknown(`known path is a ${stat.kind}, not a file`), + mtimeMs: stat.mtimeMs, + }; + }); +} + +/** Hand the event loop back between deep-scan phases. Not a throttle: the + * collectors are synchronous, and without this an embedding HTTP server + * cannot answer anything at all for the whole run. */ +const breathe = () => new Promise((resolve) => { setImmediate(resolve); }); + +function freshScanState() { + return { + running: false, + phase: 'idle', + scanned: 0, + total: 0, + path: null, + startedAt: null, + finishedAt: null, + durationMs: null, + error: null, + asOf: null, + }; +} + +/** + * Build the composed collector. One instance per dashboard server (or per CLI + * invocation) owns the TTL cache and the single-flight slot; two instances + * would defeat both, which is why this is a factory and not module state. + * + * Every collaborator is injectable so a test can drive the whole composition + * without touching the real machine — the same discipline the individual + * collectors already follow. + * + * @param {{ + * now?: () => number, ttlMs?: number, staleAfterMs?: number, + * snapshotFile?: string, fsImpl?: typeof fs, cwd?: string, + * loadConfig?: () => object, + * discoverProjects?: () => Array<{ path: string, label: string }>, + * collectors?: Record, collectorOptions?: Record, + * readSnapshotImpl?: typeof readSnapshot, writeSnapshotImpl?: typeof writeSnapshot, + * }} [options] + */ +export function createSystemCollector({ + now = Date.now, + ttlMs = CHEAP_TTL_MS, + staleAfterMs = SNAPSHOT_STALE_AFTER_MS, + snapshotFile = null, + fsImpl = fs, + cwd = process.cwd(), + loadConfig = loadKitConfig, + discoverProjects = discoverRuvfloProjects, + collectors = {}, + collectorOptions = {}, + readSnapshotImpl = readSnapshot, + writeSnapshotImpl = writeSnapshot, +} = {}) { + const collect = { + runtime: collectRuntimeCensus, + install: collectInstall, + storage: collectStorage, + catalog: collectCatalog, + projects: collectProjects, + ...collectors, + }; + const snapshotOpts = { ...(snapshotFile ? { file: snapshotFile } : {}), fsImpl }; + + /** @type {{ at: number, runtime: any, knownFiles: object, + * snapshot: ReturnType, + * sections: Record, freshness: object }|null} */ + let cheap = null; + /** @type {Promise|null} — the single-flight slot (invariant 7). */ + let inFlight = null; + let scan = freshScanState(); + + const markPhase = (phase, extra = {}) => { + scan = { ...scan, phase, ...extra }; + }; + + /** The cheap tier, memoized for `ttlMs`. Invalidated (not merely aged out) + * the moment a deep scan lands, so a completed rescan is visible on the + * very next read rather than up to a minute later. */ + async function cheapTier() { + const at = now(); + if (cheap && at - cheap.at <= ttlMs) return cheap; + + // The runtime census never throws by contract; guard anyway so a future + // change there cannot take the whole payload down with it. + let runtime; + try { + runtime = await collect.runtime({ cwd, ...(collectorOptions.runtime ?? {}) }); + } catch (error) { + runtime = { error: String(error?.code || error?.message || error), processes: null }; + } + + const persisted = readSnapshotImpl(snapshotOpts); + const freshness = snapshotFreshness(persisted, { now: at, staleAfterMs }); + // Carried forward, not re-stamped as current: a figure measured last week + // is presented with last week's asOf (invariant 3). + const sections = persisted.present + ? Object.fromEntries(SNAPSHOT_SECTIONS.map((key) => ( + [key, persisted.sections?.[key] == null ? null : carryForward(persisted.sections[key], persisted.asOf)] + ))) + : Object.fromEntries(SNAPSHOT_SECTIONS.map((key) => [key, null])); + + cheap = { + at, + runtime, + knownFiles: { asOf: at, nodes: knownFileNodes({ asOf: at, fsImpl }) }, + snapshot: persisted, + sections, + freshness, + }; + return cheap; + } + + /** Assemble the wire payload. The scan block is read LIVE (never from the + * TTL cache) so a rescan started microseconds ago already reads as running. + */ + async function read() { + const tier = await cheapTier(); + const { file, present, reason, completeness } = tier.snapshot; + return { + generatedAt: new Date(now()).toISOString(), + platform: process.platform, + // The census is ephemeral by invariant 5 — computed per request (within + // the TTL window), never written to the snapshot file. + runtime: tier.runtime, + knownFiles: tier.knownFiles, + ...tier.sections, + snapshot: { + present, + file, + reason, + completeness, + ...tier.freshness, + }, + cheapTier: { asOf: tier.at, ttlMs }, + scan: { ...scan }, + }; + } + + /** Run the four deep collectors in order, persist, invalidate the cheap + * cache. Never rejects: a scan that blows up records the reason in `scan` + * and resolves with `ok: false`, so a fire-and-forget caller (the HTTP + * route) cannot produce an unhandled rejection and a waiting caller (the + * CLI) still gets an answer. */ + async function runDeep() { + const startedAt = now(); + scan = { ...freshScanState(), running: true, phase: 'install', startedAt, asOf: startedAt }; + const sections = {}; + try { + // Yield BEFORE the first synchronous collector. `refreshDeep()` runs this + // body up to the first await, so without this an HTTP caller that starts + // a scan would wait out the install walk before its own response — the + // start-or-attach route must return while the scan runs, not after it. + await breathe(); + + // Discovery is a candidate-path source shared by two collectors + // (invariant 9). Resolve it ONCE so storage's learning-store nodes and + // the Projects table describe the same set of projects. + let catalog = null; + try { catalog = discoverProjects(); } catch { catalog = null; } + const projectPaths = Array.isArray(catalog) ? catalog.map((p) => p.path) : null; + + let cfg = {}; + try { cfg = loadConfig() ?? {}; } catch { cfg = {}; } + + sections.install = collect.install({ now: () => startedAt, fsImpl, ...(collectorOptions.install ?? {}) }); + await breathe(); + + markPhase('storage'); + // `projects: null` is load-bearing — it means "no catalog was supplied", + // which storage reports as unknown rather than a fabricated zero. + sections.storage = collect.storage({ + projects: projectPaths, now: () => startedAt, fsImpl, ...(collectorOptions.storage ?? {}), + }); + await breathe(); + + markPhase('catalog'); + sections.catalog = collect.catalog({ cwd, cfg, now: () => startedAt, fsImpl, ...(collectorOptions.catalog ?? {}) }); + await breathe(); + + markPhase('projects', { scanned: 0, total: Array.isArray(catalog) ? catalog.length : 0 }); + sections.projects = collect.projects({ + projects: catalog, + now: () => startedAt, + fsImpl, + onProgress: ({ scanned, total, path: at }) => { + scan = { ...scan, scanned, total, path: at ?? null }; + }, + ...(collectorOptions.projects ?? {}), + }); + await breathe(); + + markPhase('persist'); + const persisted = writeSnapshotImpl(sections, { ...snapshotOpts, now: now(), asOf: startedAt }); + const finishedAt = now(); + scan = { + ...scan, + running: false, + phase: 'done', + finishedAt, + durationMs: finishedAt - startedAt, + // A snapshot that could not be written is a degraded convenience, not a + // failed scan: the figures were still measured. Say so without + // pretending the scan failed. + error: persisted.ok ? null : `snapshot not persisted: ${persisted.error}`, + }; + cheap = null; + return { + ok: true, + asOf: startedAt, + sections, + completeness: summarizeCompleteness(sections), + persisted, + error: scan.error, + }; + } catch (error) { + const finishedAt = now(); + const reason = String(error?.code || error?.message || error); + scan = { + ...scan, + running: false, + phase: 'failed', + finishedAt, + durationMs: finishedAt - startedAt, + error: reason, + }; + cheap = null; + // Whatever DID complete is returned rather than discarded — one failed + // section must not erase three measured ones. + return { + ok: false, + asOf: startedAt, + sections, + completeness: summarizeCompleteness(sections), + persisted: null, + error: reason, + }; + } + } + + return { + read, + + /** Start the deep scan, or attach to the one already running. Both callers + * get the SAME promise, so two concurrent refreshes can never race each + * other or double-write the snapshot (invariant 7). */ + refreshDeep() { + if (inFlight) return inFlight; + inFlight = runDeep().finally(() => { inFlight = null; }); + return inFlight; + }, + + /** Live progress, cheap enough to read on every request. */ + scanState() { return { ...scan }; }, + + /** True while a deep scan holds the single-flight slot. */ + isScanning() { return inFlight != null; }, + + /** Drop the TTL cache (tests, and after anything that invalidates the + * cheap tier out-of-band). The single-flight slot is deliberately NOT + * cleared — a running scan owns it until it settles. */ + invalidate() { cheap = null; }, + }; +} diff --git a/src/lib/footprint/install.mjs b/src/lib/footprint/install.mjs new file mode 100644 index 0000000..c5b5e53 --- /dev/null +++ b/src/lib/footprint/install.mjs @@ -0,0 +1,467 @@ +// Install footprint — one HostInstallation per managed tool, the shared caches +// that sit next to them, the duplicate native builds nobody can see today, and +// the free-space denominator that keeps "3.8 GB" honest (ADR-0025 §4, +// docs/ddd/machine-footprint.md "Install footprint"). +// +// The managed-tool list is DERIVED, not hand-maintained. There is no single +// upstream array of "everything ak manages": HOST_REGISTRY owns the frontier +// CLIs, ruflo/agentic-qe are npm globals named in versions.mjs, the brain KB is +// a filesystem install with its own module, and the kit is its own package. A +// fourth hand-written list would drift the moment a host is added, so this +// module composes those four authorities instead. +// +// Install METHOD granularity did not exist before this module: hostInstallState +// answers npm / external / absent only. Attribution here is a pure filesystem +// probe — resolve the bin on PATH, realpath it, match the resolved path against +// known manager prefixes — and it fails closed to 'external' ("detected, not +// attributable") rather than guessing. npm containment is tested FIRST because +// npm's own prefix is frequently inside a version manager (mise on this +// machine): an npm-installed package must not be misread as mise-installed. +// +// Everything here is metadata: dirents, lstat, statfs, and package.json +// version/name fields. No tool's source or data is ever read. +import fs from 'node:fs'; +import path from 'node:path'; +import { HOST_REGISTRY } from '../adapters/registries.mjs'; +import { + home, isWindows, globalRoot, npxCacheDir, claudeDir, codexPluginCacheDir, +} from '../paths.mjs'; +import { installedVersion, KIT_PKG } from '../versions.mjs'; +import { kbDir, present as brainPresent, installedVersion as brainVersion } from '../ruvnet-brain.mjs'; +import { readJson } from '../settings.mjs'; +import { + walkTree, walkMeasurements, rootMeasurements, measured, unknown, sumMeasurements, statNode, +} from './walk.mjs'; + +/** Install-method vocabulary. 'external' is the fail-closed value: the tool is + * really there, ak did not put it there, and the manager could not be + * attributed from the resolved path. 'installer' is a tool ak installs by + * running its own installer rather than through a package manager (the brain + * KB). 'unknown' means the probe itself could not run (no npm global root, + * unreadable PATH). */ +export const INSTALL_METHODS = Object.freeze([ + 'npm', 'mise', 'asdf', 'volta', 'nvm', 'homebrew', 'system', + 'installer', 'external', 'absent', 'unknown', +]); + +// Ordered; first match wins. Deliberately conservative — a prefix that could +// belong to two managers is left to the 'external' fallthrough. +const METHOD_RULES = Object.freeze([ + { method: 'mise', rule: /[\\/]mise[\\/]/i }, + { method: 'asdf', rule: /[\\/]\.asdf[\\/]/ }, + { method: 'volta', rule: /[\\/]\.volta[\\/]/ }, + { method: 'nvm', rule: /[\\/]\.nvm[\\/]/ }, + { method: 'homebrew', rule: /[\\/](?:homebrew|Cellar|linuxbrew)[\\/]/ }, + { method: 'system', rule: /^(?:\/usr\/bin|\/bin|\/usr\/local\/bin)[\\/][^\\/]+$/ }, +]); + +/** Native addons are the sprawl this section exists to expose; a tree with + * hundreds of them is real (ruflo bundles several native stacks), but the + * payload still needs a ceiling. The COUNT stays exact past the cap. */ +export const MAX_NATIVE_ADDONS_PER_TOOL = 512; +const NATIVE_EXT = '.node'; + +/** Attribute an install from a binary's REAL path. `globalRootDir`, when known, + * wins over every manager rule: a package under npm's global node_modules is + * npm-installed no matter which version manager owns the prefix. */ +export function attributeInstallMethod(realPath, { globalRootDir = null } = {}) { + if (!realPath) return 'absent'; + const resolved = path.resolve(realPath); + if (globalRootDir && !path.relative(globalRootDir, resolved).startsWith('..')) return 'npm'; + if (/[\\/]node_modules[\\/]/.test(resolved)) return 'npm'; + for (const { method, rule } of METHOD_RULES) if (rule.test(resolved)) return method; + return 'external'; +} + +/** First `bin` on PATH, realpath-resolved, or null. Pure filesystem: `which` + * costs a process per tool and the cheap tier runs on every dashboard read. */ +export function resolveBinPath(bin, { + env = process.env, windows = isWindows, fsImpl = fs, +} = {}) { + if (!bin) return null; + const dirs = String(env.PATH || env.Path || '').split(path.delimiter).filter(Boolean); + const exts = windows + ? ['', ...String(env.PATHEXT || '.COM;.EXE;.BAT;.CMD').split(';').filter(Boolean)] + : ['']; + for (const dir of dirs) { + for (const ext of exts) { + const candidate = path.join(dir, bin + ext); + try { + const st = fsImpl.lstatSync(candidate); + if (!st.isFile() && !st.isSymbolicLink()) continue; + return fsImpl.realpathSync(candidate); + } catch { /* not here: try the next PATH entry */ } + } + } + return null; +} + +/** The npm-managed package name that owns a native addon file, derived from its + * path (`.../node_modules//build/Release/x.node`, scoped names included). + * Null when the addon lives outside any node_modules — still inventoried, just + * not attributable to a package. */ +export function nativeModuleName(file) { + const parts = String(file).split(/[\\/]+/); + const idx = parts.lastIndexOf('node_modules'); + if (idx < 0 || idx + 1 >= parts.length) return null; + const first = parts[idx + 1]; + if (first.startsWith('@') && idx + 2 < parts.length) return `${first}/${parts[idx + 2]}`; + return first; +} + +function safeGlobalRoot() { + // globalRoot() throws when npm's root cannot be determined (paths.mjs). That + // is a legitimate machine state (npm absent), not a reason to fail the whole + // System panel — every npm-rooted figure degrades to unknown instead. + try { + return { root: globalRoot(), reason: null }; + } catch { + return { root: null, reason: 'npm global root undeterminable' }; + } +} + +/** The managed-tool descriptors this collector measures, composed from the + * registry plus the modules that own the non-npm tools. `kind` says how the + * root is found, not how it was installed. */ +export function managedTools({ pkgRoot = null, globalRootDir = null } = {}) { + const npmRoot = (pkg) => (globalRootDir ? path.join(globalRootDir, pkg) : null); + const tools = [ + { id: 'ruflo', label: 'ruflo', pkg: 'ruflo', bin: 'ruflo', kind: 'npm', root: npmRoot('ruflo') }, + { + id: 'agentic-qe', label: 'agentic-qe', pkg: 'agentic-qe', bin: 'aqe', + kind: 'npm', root: npmRoot('agentic-qe'), + }, + ]; + for (const host of HOST_REGISTRY) { + tools.push({ + id: host.id, + label: host.label, + pkg: host.install.npmPackage, + bin: host.install.bin, + kind: 'npm', + root: npmRoot(host.install.npmPackage), + }); + } + tools.push({ + id: 'agentic-kit', label: 'agentic-kit', pkg: KIT_PKG, bin: 'ak', + kind: 'self', root: pkgRoot || npmRoot(KIT_PKG), + }); + tools.push({ + // Not an npm global: `npx ruvnet-brain` downloads a ~2 GB offline KB to a + // cache dir and wires a user-scope Claude Code plugin. Its bytes are the + // KB's, and its version comes from the plugin manifest, not npm. + id: 'ruvnet-brain', label: 'RuvNet Brain KB', pkg: null, bin: null, + kind: 'kb', root: kbDir(), + }); + return tools; +} + +function toolVersion(desc) { + try { + if (desc.kind === 'kb') return brainVersion() ?? null; + if (desc.kind === 'self' && desc.root) { + return readJson(path.join(desc.root, 'package.json'), {})?.version ?? null; + } + return desc.pkg ? installedVersion(desc.pkg) : null; + } catch { + return null; + } +} + +/** Resolve a tool root through a symlink, ONCE, at the root itself. The walker + * never follows links found during traversal (that is how it escapes its root + * or cycles); a root the caller named explicitly is different — a dev install + * linked into npm's global tree (`npm link`) is a symlink, and refusing it + * would report the kit's own size as unmeasurable on every maintainer machine. + * The original path is retained so the row can say where it was linked from. */ +function resolveRootPath(root, fsImpl) { + if (!root) return { root: null, linkedFrom: null }; + try { + const real = fsImpl.realpathSync(root); + return { root: real, linkedFrom: real === root ? null : root }; + } catch { + return { root, linkedFrom: null }; + } +} + +function toolPresence(desc, fsImpl) { + if (desc.kind === 'kb') { + try { return brainPresent(); } catch { return false; } + } + if (!desc.root) return false; + const manifest = statNode(path.join(desc.root, 'package.json'), { fsImpl }); + if (manifest.status === 'measured' && manifest.kind === 'file') return true; + const dir = statNode(desc.root, { fsImpl }); + return dir.status === 'measured' && dir.kind === 'dir'; +} + +/** + * One HostInstallation. Shape: + * { tool, label, package, present, version, installMethod, root, linkedFrom, + * rootReason, bytes, files, newestMtimeMs, nativeAddons[], + * nativeAddonCount, nativeAddonsTruncated, degraded[], complete } + * + * A tool that is genuinely not installed reports `measured(0)` bytes — zero is + * the true size of an absent install. A tool that IS installed but whose root + * ak cannot attribute (an externally-installed CLI: mise, brew, a native + * installer) reports unknown-with-reason, never 0: ak does not own that tree + * and refuses to claim a size for it. + */ +function collectTool(desc, ctx) { + const { asOf, walk, limits, fsImpl, maxNativeAddons } = ctx; + const { root: realRoot, linkedFrom } = resolveRootPath(desc.root, fsImpl); + const resolved = { ...desc, root: realRoot }; + const present = toolPresence(resolved, fsImpl); + const version = present ? toolVersion(resolved) : null; + const base = { + tool: desc.id, + label: desc.label, + package: desc.pkg, + present, + version, + root: realRoot, + linkedFrom, + rootReason: null, + nativeAddons: [], + nativeAddonCount: 0, + nativeAddonsTruncated: false, + newestMtimeMs: null, + degraded: [], + }; + + if (!present) { + const binPath = desc.bin ? resolveBinPath(desc.bin, { fsImpl }) : null; + if (!binPath) { + return { + ...base, installMethod: 'absent', root: null, + bytes: measured(0, { asOf }), files: measured(0, { asOf }), complete: true, + }; + } + // Installed, but by something other than ak. MANAGED-TOOLS invariant 2 + // ("honest disowning"): report what is true — the method and the binary — + // and refuse to invent a tree size for a layout ak does not own. + const reason = 'external install: tree root not attributable'; + return { + ...base, + present: true, + installMethod: attributeInstallMethod(binPath, { globalRootDir: ctx.globalRootDir }), + root: binPath, + rootReason: reason, + bytes: unknown(reason), + files: unknown(reason), + complete: false, + }; + } + + if (!realRoot) { + const reason = ctx.globalRootReason || 'install root unknown'; + return { + ...base, installMethod: 'unknown', bytes: unknown(reason), files: unknown(reason), + rootReason: reason, complete: false, + }; + } + + const addons = []; + let addonCount = 0; + const result = walk(realRoot, { + ...limits, + fsImpl, + onFile: ({ file, name, bytes, mtimeMs }) => { + if (!name.endsWith(NATIVE_EXT)) return; + addonCount += 1; + if (addons.length < maxNativeAddons) { + addons.push({ tool: desc.id, module: nativeModuleName(file), name, file, bytes, mtimeMs }); + } + }, + }); + const { bytes, files } = walkMeasurements(result, { asOf }); + return { + ...base, + installMethod: desc.kind === 'kb' ? 'installer' : 'npm', + bytes, + files, + newestMtimeMs: result.newestMtimeMs, + nativeAddons: addons, + nativeAddonCount: addonCount, + nativeAddonsTruncated: addonCount > addons.length, + degraded: result.degraded, + complete: result.complete, + }; +} + +/** The same native module compiled into more than one tree — sprawl that is + * invisible today. Grouped by module + addon filename, because two different + * addons inside one package are not duplicates of each other. `wastedBytes` is + * everything past the largest copy: the floor of what deduplication would + * return, stated conservatively. */ +export function duplicateNativeBuilds(addons) { + const groups = new Map(); + for (const addon of addons || []) { + const key = `${addon.module ?? '(outside node_modules)'}::${addon.name}`; + if (!groups.has(key)) { + groups.set(key, { module: addon.module ?? null, addon: addon.name, copies: [] }); + } + const group = groups.get(key); + if (!group.copies.some((c) => c.file === addon.file)) { + group.copies.push({ tool: addon.tool, file: addon.file, bytes: addon.bytes }); + } + } + return [...groups.values()] + .filter((g) => g.copies.length > 1) + .map((g) => { + const total = g.copies.reduce((acc, c) => acc + c.bytes, 0); + const largest = g.copies.reduce((acc, c) => Math.max(acc, c.bytes), 0); + return { ...g, copyCount: g.copies.length, totalBytes: total, wastedBytes: total - largest }; + }) + .sort((a, b) => b.wastedBytes - a.wastedBytes); +} + +/** One node per npx cache env (`/_npx/`). Exported because the + * storage collector's reclaimable rows need exactly these figures — walking + * the cache twice would double the I/O to answer one question. Package NAMES + * come from the env's own package.json manifest; nothing else is read. */ +export function npxEnvNodes({ + root = npxCacheDir(), walk = walkTree, limits = {}, asOf = null, fsImpl = fs, +} = {}) { + let entries; + try { + entries = fsImpl.readdirSync(root, { withFileTypes: true }); + } catch (err) { + const code = err?.code || 'io'; + return { + root, + presence: code === 'ENOENT' ? 'absent' : 'degraded', + reason: code === 'ENOENT' ? null : code, + envs: [], + }; + } + const envs = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.isSymbolicLink()) continue; + const dir = path.join(root, entry.name); + const result = walk(dir, { ...limits, fsImpl }); + const { bytes, files } = walkMeasurements(result, { asOf }); + const manifest = readJson(path.join(dir, 'package.json'), {}) ?? {}; + envs.push({ + id: entry.name, + path: dir, + packages: Object.keys(manifest.dependencies ?? {}), + bytes, + files, + newestMtimeMs: result.newestMtimeMs, + complete: result.complete, + }); + } + envs.sort((a, b) => (b.bytes.value ?? 0) - (a.bytes.value ?? 0)); + return { root, presence: 'present', reason: null, envs }; +} + +/** Install-adjacent shared caches. Deliberately their OWN rows rather than + * smeared into a tool's tree: npx envs and browser binaries belong to no + * single tool, and hiding them inside one would misattribute the bytes. The + * brain KB is absent here on purpose — it is a managed tool with its own row. + * Windows browser caches live under LOCALAPPDATA; both candidates are listed + * and the platform's absent one reads as a measured zero. */ +export function sharedCacheRoots({ env = process.env } = {}) { + const localAppData = env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'); + return [ + { id: 'npx-envs', label: 'npx cache envs', path: npxCacheDir() }, + { id: 'claude-plugins', label: 'Claude Code plugins', path: path.join(claudeDir(), 'plugins') }, + { id: 'codex-plugins', label: 'Codex plugin cache', path: codexPluginCacheDir() }, + { id: 'playwright', label: 'Playwright browsers', path: path.join(home, '.cache', 'ms-playwright') }, + { id: 'playwright-win', label: 'Playwright browsers', path: path.join(localAppData, 'ms-playwright') }, + { id: 'puppeteer', label: 'Puppeteer browsers', path: path.join(home, '.cache', 'puppeteer') }, + ]; +} + +/** The section's denominator: "the install is X GB" is meaningless without an + * "of Y free" beside it. statfs is a Node builtin (>=18.15) — no dependency, + * and it works on Windows too. Failure reports unknown, never 0. */ +export function diskSpace(target = home, { fsImpl = fs } = {}) { + try { + const st = fsImpl.statfsSync(target); + const block = Number(st.bsize); + return { + path: target, + totalBytes: measured(block * Number(st.blocks)), + freeBytes: measured(block * Number(st.bavail)), + }; + } catch (err) { + const reason = err?.code || 'statfs unavailable'; + return { path: target, totalBytes: unknown(reason), freeBytes: unknown(reason) }; + } +} + +/** + * The install section of a FootprintSnapshot. + * + * @param {{ + * pkgRoot?: string|null, now?: () => number, walk?: typeof walkTree, + * limits?: object, diskPath?: string, maxNativeAddons?: number, + * includeCaches?: boolean, fsImpl?: typeof fs, + * }} [options] + * @returns {{ + * asOf: number, globalRoot: string|null, globalRootReason: string|null, + * tools: object[], sharedCaches: object[], npxEnvs: object, + * duplicateNatives: object[], totals: object, disk: object, complete: boolean, + * }} + */ +export function collectInstall({ + pkgRoot = null, + now = Date.now, + walk = walkTree, + limits = {}, + diskPath = home, + maxNativeAddons = MAX_NATIVE_ADDONS_PER_TOOL, + includeCaches = true, + fsImpl = fs, +} = {}) { + const asOf = now(); + const { root: globalRootDir, reason: globalRootReason } = safeGlobalRoot(); + const ctx = { asOf, walk, limits, fsImpl, maxNativeAddons, globalRootDir, globalRootReason }; + + const tools = managedTools({ pkgRoot, globalRootDir }).map((desc) => collectTool(desc, ctx)); + + const sharedCaches = []; + let npxEnvs = { root: npxCacheDir(), presence: 'unknown', reason: 'not collected', envs: [] }; + if (includeCaches) { + for (const cache of sharedCacheRoots()) { + const result = walk(cache.path, { ...limits, fsImpl }); + const { presence, bytes, files } = rootMeasurements(result, { asOf }); + sharedCaches.push({ + ...cache, presence, bytes, files, newestMtimeMs: result.newestMtimeMs, + complete: result.complete, + }); + } + npxEnvs = npxEnvNodes({ walk, limits, asOf, fsImpl }); + } + + const allAddons = tools.flatMap((t) => t.nativeAddons); + const toolBytes = tools.map((t) => t.bytes); + const cacheBytes = sharedCaches.map((c) => c.bytes); + const complete = tools.every((t) => t.complete !== false) + && sharedCaches.every((c) => c.complete !== false) + && !tools.some((t) => t.nativeAddonsTruncated); + + return { + asOf, + globalRoot: globalRootDir, + globalRootReason, + tools, + sharedCaches, + npxEnvs, + duplicateNatives: duplicateNativeBuilds(allAddons), + totals: { + installBytes: sumMeasurements(toolBytes, { asOf }), + cacheBytes: sumMeasurements(cacheBytes, { asOf }), + // Counts, not bytes: a truncated addon list still knows how many it saw. + nativeAddons: measured( + tools.reduce((acc, t) => acc + t.nativeAddonCount, 0), + { asOf, partial: tools.some((t) => t.complete === false) }, + ), + toolsPresent: measured(tools.filter((t) => t.present).length, { asOf }), + }, + disk: diskSpace(diskPath, { fsImpl }), + complete, + }; +} diff --git a/src/lib/footprint/projects.mjs b/src/lib/footprint/projects.mjs new file mode 100644 index 0000000..b83fc98 --- /dev/null +++ b/src/lib/footprint/projects.mjs @@ -0,0 +1,457 @@ +// Project footprints — one row per project in the shared discovery catalog: +// approximate LOC by language, working-tree bytes, `.git` bytes, `node_modules` +// bytes, last activity, and the origin remote's web page when one exists. +// +// The three byte figures stay SEPARATE on purpose (ADR-0025): `node_modules` +// dominates and `.git` distorts, so folding either into the tree would let +// reinstallable overhead masquerade as "your project got big". +// +// LOC is APPROXIMATE by invariant 11 and says so in the payload: it is an +// extension-bucketed newline count with a stated exclusion list, produced by the +// kit's own bounded walker (zero runtime dependencies — no cloc, no tokei). Files +// are scanned through a fixed 64 KB buffer that is counted and discarded; no file +// content is ever retained or emitted. +// +// Discovery supplies PATHS ONLY (invariant 9). Every figure below is measured here. +import fs from 'node:fs'; +import path from 'node:path'; +import { parseRepoSlug } from '../admin-collect.mjs'; +import { discoverRuvfloProjects } from '../dashboard/project-discovery.mjs'; +import { walkTree, rootMeasurements, measured, unknown, sumMeasurements } from './walk.mjs'; + +/** Directories that are never code and never the user's work. Excluded from the + * tree walk, from LOC, and from the nested-`node_modules` search alike, so the + * three byte figures partition the project rather than overlapping it. */ +const OVERHEAD_DIRS = new Set(['.git', 'node_modules']); + +/** Vendored / generated / virtual-env trees. Excluded from LOC only: they are + * real bytes on disk (so they stay in treeBytes) but they are not lines the user + * wrote, and counting them would make the LOC figure meaningless. */ +const LOC_EXCLUDED_DIRS = new Set([ + 'node_modules', '.git', 'vendor', 'third_party', 'thirdparty', 'bower_components', + 'dist', 'build', 'out', 'target', '.next', '.nuxt', '.svelte-kit', 'coverage', + '.venv', 'venv', '__pycache__', '.tox', '.mypy_cache', '.pytest_cache', + '.gradle', '.idea', '.vscode', 'Pods', '.terraform', '.cache', '.turbo', +]); + +/** Machine-generated manifests: text, enormous, and nobody's line count. */ +const LOC_EXCLUDED_FILES = new Set([ + 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'npm-shrinkwrap.json', + 'Cargo.lock', 'poetry.lock', 'Gemfile.lock', 'composer.lock', 'go.sum', 'flake.lock', +]); + +/** Extension → language bucket. An extension absent from this map is NOT counted + * — an unknown extension may be a binary, and guessing would inflate the total. */ +const LANGUAGES = new Map(Object.entries({ + '.js': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript', '.jsx': 'javascript', + '.ts': 'typescript', '.tsx': 'typescript', '.mts': 'typescript', '.cts': 'typescript', + '.rs': 'rust', '.go': 'go', '.py': 'python', '.rb': 'ruby', '.php': 'php', + '.java': 'java', '.kt': 'kotlin', '.kts': 'kotlin', '.scala': 'scala', '.groovy': 'groovy', + '.cs': 'csharp', '.fs': 'fsharp', '.swift': 'swift', '.m': 'objective-c', '.mm': 'objective-c', + '.c': 'c', '.h': 'c', '.cc': 'cpp', '.cpp': 'cpp', '.cxx': 'cpp', '.hpp': 'cpp', '.hh': 'cpp', + '.ex': 'elixir', '.exs': 'elixir', '.erl': 'erlang', '.hs': 'haskell', '.clj': 'clojure', + '.lua': 'lua', '.dart': 'dart', '.zig': 'zig', '.nim': 'nim', '.pl': 'perl', '.r': 'r', + '.sh': 'shell', '.bash': 'shell', '.zsh': 'shell', '.fish': 'shell', '.ps1': 'powershell', + '.sql': 'sql', '.vue': 'vue', '.svelte': 'svelte', + '.html': 'html', '.htm': 'html', '.css': 'css', '.scss': 'css', '.sass': 'css', '.less': 'css', + '.md': 'markdown', '.mdx': 'markdown', + '.json': 'config', '.yml': 'config', '.yaml': 'config', '.toml': 'config', + '.ini': 'config', '.xml': 'config', '.proto': 'config', '.tf': 'config', +})); + +/** Source trees nest far shallower than dependency trees; 12 covers a monorepo + * package's deepest source dir without letting a pathological tree run away. */ +const LOC_MAX_DEPTH = 12; +/** Files above this are minified bundles, fixtures, or data dumps far more often + * than they are hand-written source. Skipped and reported, not counted. */ +const LOC_MAX_FILE_BYTES = 2 * 1024 * 1024; +const READ_CHUNK = 64 * 1024; +/** Depth at which a workspace's nested `node_modules` still gets attributed. */ +const NODE_MODULES_MAX_DEPTH = 6; + +/** The exclusion list every consumer must be able to state alongside the figure + * (invariant 11). Deliberate exclusions are why LOC is approximate; they are NOT + * a failed measurement, so they never mark the count partial. */ +export const LOC_EXCLUSIONS = Object.freeze([ + ...[...LOC_EXCLUDED_DIRS].sort().map((dir) => `${dir}/`), + ...[...LOC_EXCLUDED_FILES].sort(), + 'files without a recognized source extension', + 'files containing NUL bytes (binary)', + `files larger than ${LOC_MAX_FILE_BYTES} bytes`, +]); + +// ── git remote ──────────────────────────────────────────────────────────────── + +/** + * The `url` of a named remote in `.git/config` text. Ordered so `origin` always + * wins; a repository with remotes but no `origin` still reports its remote rather + * than claiming to be local-only, which would be a false negative. + * + * @param {string} source raw `.git/config` contents + * @returns {{ name: string, url: string }|null} + */ +export function parseGitRemote(source) { + const found = new Map(); + const section = /^\s*\[\s*remote\s+"([^"]+)"\s*\]\s*$/gm; + let match; + while ((match = section.exec(source)) !== null) { + const rest = source.slice(section.lastIndex); + const nextSection = rest.search(/^\s*\[/m); + const body = nextSection < 0 ? rest : rest.slice(0, nextSection); + const url = body.match(/^\s*url\s*=\s*(.+?)\s*$/m); + if (url?.[1]) found.set(match[1], url[1]); + } + if (found.has('origin')) return { name: 'origin', url: found.get('origin') }; + const first = [...found.entries()][0]; + return first ? { name: first[0], url: first[1] } : null; +} + +/** Forges whose repository page is `https:////`. */ +const KNOWN_FORGES = new Map(Object.entries({ + 'github.com': 'github', + 'gitlab.com': 'gitlab', + 'bitbucket.org': 'bitbucket', + 'codeberg.org': 'codeberg', + 'git.sr.ht': 'sourcehut', +})); + +/** Hostname + scheme from any of the shapes parseRepoSlug already proves: + * git+https, ssh://, scp-shorthand, bare https. */ +function remoteHost(rawUrl) { + let s = rawUrl.trim(); + if (s.startsWith('git+')) s = s.slice(4); + const scp = s.match(/^[^@/]+@([^:/]+):(.+)$/); // git@github.com:owner/repo + if (scp) return { hostname: scp[1].toLowerCase(), scheme: null }; + try { + const url = new URL(s); + const scheme = url.protocol === 'https:' || url.protocol === 'http:' ? url.protocol.slice(0, -1) : null; + return { hostname: url.hostname.toLowerCase().replace(/^www\./, ''), scheme }; + } catch { return { hostname: null, scheme: null }; } +} + +/** + * Describe a remote URL for the Projects table. A recognized forge or an + * already-web-shaped self-hosted URL yields a `webUrl`; anything else yields the + * remote unlinked. The URL is NEVER guessed and the kit never fetches it — the + * link exists so the user's own browser can navigate there. + * + * @param {string} rawUrl + * @param {string} [name] the remote's name (`origin`) + * @returns {{ status: 'linked'|'unrecognized', name: string, raw: string, + * hostname: string|null, host: string|null, slug: string|null, + * webUrl: string|null }} + */ +export function describeRemote(rawUrl, name = 'origin') { + const raw = String(rawUrl ?? '').trim(); + const { hostname, scheme } = remoteHost(raw); + const slug = parseRepoSlug(raw); + const forge = hostname ? KNOWN_FORGES.get(hostname) : null; + const linkable = Boolean(slug && hostname && (forge || scheme)); + // A known forge is https by definition; a self-hosted remote keeps the scheme it + // was already written with, so an http-only server is not silently upgraded. + const webScheme = forge ? 'https' : scheme; + return { + status: linkable ? 'linked' : 'unrecognized', + name, + raw, + hostname, + host: forge ?? hostname, + slug, + webUrl: linkable ? `${webScheme}://${hostname}/${slug}` : null, + }; +} + +/** + * The remote row for one project. Absence is stated, never guessed: no `.git` or + * no configured remote is an explicit `local-only`; an unreadable `.git/config` + * is `unknown` with its errno, not a silent local-only. + */ +export function projectRemote(projectPath, { fsImpl = fs } = {}) { + const configFile = path.join(projectPath, '.git', 'config'); + let source; + try { source = fsImpl.readFileSync(configFile, 'utf8'); } catch (error) { + if (error.code === 'ENOENT' || error.code === 'ENOTDIR') { + return { status: 'local-only', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: null }; + } + return { status: 'unknown', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: error.code ?? 'EUNKNOWN' }; + } + const remote = parseGitRemote(source); + if (!remote) { + return { status: 'local-only', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: null }; + } + return { ...describeRemote(remote.url, remote.name), reason: null }; +} + +// ── lines of code ───────────────────────────────────────────────────────────── + +/** Count newlines in one file through a fixed buffer. Returns null when the file + * is binary (a NUL byte in the first chunk) or unreadable — never 0, which would + * claim an empty file. Each chunk is counted and immediately overwritten; no file + * content is retained past this function or emitted anywhere. */ +function countFileLines(file, size, fsImpl = fs) { + let fd; + try { fd = fsImpl.openSync(file, 'r'); } catch { return null; } + try { + const buffer = Buffer.allocUnsafe(READ_CHUNK); + let lines = 0; + let read = 0; + let lastByte = 0; + let offset = 0; + let first = true; + while ((read = fsImpl.readSync(fd, buffer, 0, READ_CHUNK, offset)) > 0) { + // One NUL in the first chunk is the cheap, conventional binary test. + if (first) { const nul = buffer.indexOf(0); if (nul >= 0 && nul < read) return null; } + first = false; + for (let i = 0; i < read; i++) if (buffer[i] === 0x0a) lines++; + lastByte = buffer[read - 1]; + offset += read; + } + // A final line with no trailing newline still counts as a line. + if (size > 0 && lastByte !== 0x0a) lines++; + return lines; + } catch { return null; } + finally { try { fsImpl.closeSync(fd); } catch { /* fd already gone */ } } +} + +/** + * Approximate lines of code under `root`, bucketed by language, via the shared + * bounded walker — so LOC inherits the same never-follow-symlinks rule, entry + * caps, and degrade-this-node-only failure mode as every other figure here. + * + * `total` is `partial` only when a cap fired or a subtree was unreadable. The + * exclusion list is a deliberate scope, not a failure, and never marks it partial + * — which is precisely why the figure ships with `approximate` and `exclusions` + * attached, so no consumer can render it as authoritative (invariant 11). + * + * @param {string} root + * @param {{ walk?: Function, limits?: object, maxDepth?: number, asOf?: number|null, + * fsImpl?: typeof fs }} [options] + * @returns {object} LocCount + */ +export function countLines(root, { + walk = walkTree, limits = {}, maxDepth = LOC_MAX_DEPTH, asOf = null, fsImpl = fs, +} = {}) { + const byLanguage = {}; + let total = 0; + let files = 0; + let skipped = 0; + const languageOf = (name) => LANGUAGES.get(path.extname(name).toLowerCase()); + + const result = walk(root, { + maxDepth, ...limits, fsImpl, + skipDir: (dir, name) => LOC_EXCLUDED_DIRS.has(name), + acceptFile: (name) => !LOC_EXCLUDED_FILES.has(name) && Boolean(languageOf(name)), + onFile: ({ file, name, bytes }) => { + if (bytes > LOC_MAX_FILE_BYTES) { skipped++; return; } + const lines = countFileLines(file, bytes, fsImpl); + if (lines === null) { skipped++; return; } + const language = languageOf(name); + byLanguage[language] = (byLanguage[language] ?? 0) + lines; + total += lines; + files++; + }, + }); + + const base = { approximate: true, exclusions: [...LOC_EXCLUSIONS] }; + if (result.status === 'unknown') { + return { + ...base, total: unknown(result.reason), byLanguage: {}, files: null, skipped: 0, + complete: false, degraded: result.degraded ?? [], + }; + } + return { + ...base, + total: measured(total, { asOf, partial: result.complete === false }), + byLanguage, + files, + skipped, + complete: result.complete !== false, + degraded: result.degraded ?? [], + }; +} + +// ── bytes ───────────────────────────────────────────────────────────────────── + +/** One walked root as Measurements plus its newest mtime. `rootMeasurements` + * already draws the absent-vs-degraded line: a directory that does not exist + * holds a real, measured zero; one that could not be read stays unknown. */ +function walkNode(walk, root, options) { + const result = walk(root, options); + return { + ...rootMeasurements(result, { asOf: options.asOf ?? null }), + newestMtimeMs: Number.isFinite(result.newestMtimeMs) ? result.newestMtimeMs : null, + complete: result.complete !== false, + }; +} + +/** + * Top-most `node_modules` directories under `root`, bounded by depth. Nested + * copies inside a found one are its own bytes, so the search never descends into + * a hit — the roots returned partition rather than overlap. + * + * @param {string} root + * @param {{ walk?: Function, maxDepth?: number, fsImpl?: typeof fs }} [options] + * @returns {string[]} + */ +export function nodeModulesRoots(root, { walk = walkTree, maxDepth = NODE_MODULES_MAX_DEPTH, fsImpl = fs } = {}) { + const roots = []; + // `skipDir` is the walker's directory hook: recording a hit and pruning it in + // one step is what keeps the roots non-overlapping, so their bytes sum cleanly. + walk(root, { + maxDepth, fsImpl, + acceptFile: () => false, // directories are the subject here; no file work + skipDir: (dir, name) => { + if (name === 'node_modules') { if (roots.length < 256) roots.push(dir); return true; } + return OVERHEAD_DIRS.has(name) || name.startsWith('.'); + }, + }); + return roots; +} + +// ── assembly ────────────────────────────────────────────────────────────────── + +/** The row for a project whose path could not be measured at all. Every figure is + * unknown-with-reason; none of them is 0, because nothing was measured. */ +function missingProject(project, reason, presence = 'absent') { + return { + path: project.path, + label: project.label, + source: project.source ?? null, + remote: { status: 'unknown', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason }, + loc: { approximate: true, exclusions: [...LOC_EXCLUSIONS], total: unknown(reason), + byLanguage: {}, files: null, skipped: 0, complete: false, degraded: [] }, + presence, + treeBytes: unknown(reason), + treeFiles: unknown(reason), + gitBytes: unknown(reason), + nodeModulesBytes: unknown(reason), + nodeModulesRoots: [], + totalBytes: unknown(reason), + lastActivity: unknown(reason), + treeExclusions: [...OVERHEAD_DIRS], + complete: false, + }; +} + +/** Progress callbacks belong to the UI; a throwing one is the UI's problem, not + * the scan's (the same contract usage-index's notify() keeps). */ +function notify(onProgress, payload) { + if (typeof onProgress !== 'function') return; + try { onProgress(payload); } catch { /* never let a listener abort a scan */ } +} + +/** + * Measure one project. `walk` is the shared bounded walker; the project's path is + * the only thing discovery contributes (invariant 9). + * + * @param {{ path: string, label: string, source?: string }} project + * @param {{ walk?: Function, limits?: object, countLines?: Function, loc?: boolean, + * asOf?: number|null, fsImpl?: typeof fs }} [options] + * @returns {object} ProjectFootprint + */ +export function measureProject(project, { + walk = walkTree, limits = {}, countLines: countLinesImpl = countLines, + loc = true, asOf = null, fsImpl = fs, +} = {}) { + const root = project.path; + const common = { ...limits, fsImpl, asOf }; + const tree = walkNode(walk, root, { ...common, skipDir: (dir, name) => OVERHEAD_DIRS.has(name) }); + + // A project whose ROOT is gone or unreadable is not a project measuring zero + // bytes — it is a project we could not measure. `rootMeasurements` turns an + // absent root into a real 0, which is right for a missing `.git` or + // `node_modules` and wrong for the project itself, so it is overridden here. + if (tree.presence !== 'present') { + // The remote is not probed either: a vanished path has no `.git/config`, and + // reporting that as "local only" would invent a fact about a project we + // cannot see. + return missingProject(project, tree.presence === 'absent' + ? 'project path no longer exists' + : 'project root unreadable', tree.presence); + } + + const git = walkNode(walk, path.join(root, '.git'), common); + + const moduleRoots = nodeModulesRoots(root, { walk, fsImpl }); + // An empty roots list is a real, measured zero — this project has no + // node_modules — which is why it is stated explicitly rather than handed to + // sumMeasurements, whose empty-list zero would mean the same thing by accident. + const nodeModulesBytes = moduleRoots.length === 0 + ? measured(0, { asOf }) + : sumMeasurements(moduleRoots.map((dir) => walkNode(walk, dir, common).bytes), { asOf }); + + return { + path: root, + label: project.label, + source: project.source ?? null, + remote: projectRemote(root, { fsImpl }), + loc: loc + ? countLinesImpl(root, { walk, limits, asOf, fsImpl }) + : { approximate: true, exclusions: [...LOC_EXCLUSIONS], total: unknown('not measured'), + byLanguage: {}, files: null, skipped: 0, complete: false, degraded: [] }, + presence: tree.presence, + treeBytes: tree.bytes, + treeFiles: tree.files, + gitBytes: git.bytes, + nodeModulesBytes, + nodeModulesRoots: moduleRoots, + totalBytes: sumMeasurements([tree.bytes, git.bytes, nodeModulesBytes], { asOf }), + // Working-tree mtime only: `.git` and `node_modules` churn on operations the + // user did not perform, so including them would report a `pnpm install` as + // "last active". + lastActivity: tree.newestMtimeMs === null + ? unknown('no readable working-tree entry') + : measured(tree.newestMtimeMs, { asOf }), + treeExclusions: [...OVERHEAD_DIRS], + complete: tree.complete && git.complete, + }; +} + +/** + * ProjectFootprint rows for every project in the shared discovery catalog. + * + * @param {{ discover?: Function, walk?: Function, limits?: object, countLines?: Function, + * projects?: Array<{path: string, label: string}>|null, loc?: boolean, + * limit?: number|null, onProgress?: Function, now?: () => number, + * fsImpl?: typeof fs }} [options] + * @returns {object} the ProjectFootprint section of a FootprintSnapshot + */ +export function collectProjects({ + discover = discoverRuvfloProjects, + walk = walkTree, + limits = {}, + countLines: countLinesImpl = countLines, + projects = null, + loc = true, + limit = null, + onProgress = null, + now = Date.now, + fsImpl = fs, +} = {}) { + const asOf = now(); + let catalog; + // Discovery is a candidate-path source; if it cannot run, this section reports + // nothing rather than taking the rest of the snapshot down with it. + let discoveryReason = null; + try { catalog = projects ?? discover(); } catch (error) { catalog = []; discoveryReason = error?.code ?? 'discovery failed'; } + const rows = Array.isArray(catalog) ? catalog : []; + const selected = typeof limit === 'number' && limit >= 0 ? rows.slice(0, limit) : rows; + + const out = []; + for (const project of selected) { + if (!project?.path) continue; + notify(onProgress, { scanned: out.length, total: selected.length, phase: 'project', path: project.path }); + out.push(measureProject(project, { walk, limits, countLines: countLinesImpl, loc, asOf, fsImpl })); + } + notify(onProgress, { scanned: out.length, total: selected.length, phase: 'done', path: null }); + + return { + asOf, + projects: out, + count: discoveryReason ? unknown(discoveryReason) : measured(rows.length, { asOf }), + scanned: out.length, + truncated: selected.length < rows.length, + locMeasured: loc, + complete: !discoveryReason && selected.length === rows.length && out.every((row) => row.complete), + }; +} diff --git a/src/lib/footprint/runtime.mjs b/src/lib/footprint/runtime.mjs new file mode 100644 index 0000000..6d280b7 --- /dev/null +++ b/src/lib/footprint/runtime.mjs @@ -0,0 +1,204 @@ +// Machine footprint — RuntimeCensus (ADR-0025 §4, docs/ddd/machine-footprint.md). +// +// EPHEMERAL BY INVARIANT (#5): computed per request, never written into the +// persisted snapshot. A process table is a moment, not a fact worth retaining, +// and replaying a stale one as liveness is the trap that invariant closes. +// Nothing in this module writes to disk — there is no `carriedForward` figure +// here, by construction. +// +// Figures report in walk.mjs's Measurement vocabulary so an unmeasured quantity +// can never be mistaken for a measured zero (invariant #2, ADR-0023). No +// running daemons is `measured(0)`; a daemon census that failed is `unknown`. +import os from 'node:os'; +import { listDaemons, staleDaemons } from '../daemons.mjs'; +import { surveyHostProcesses } from '../live/process-sessions.mjs'; +import { resolveProjectIdentity } from '../live/project-label.mjs'; +import { measured, sumMeasurements, unknown } from './walk.mjs'; + +// Why a working directory was unavailable, in the words the Runtime table +// renders. Each is a distinguishable cause — a blocked probe, a denied handle, +// a bitness mismatch — because "we could not attribute this process" and "this +// process has no project" are different facts. A row never guesses a project +// and never renders blank. +const CWD_REASONS = new Map([ + ['cwd-unavailable', 'not attributable — the process reported no working directory'], + ['cwd-survey-failed', 'not attributable — the working-directory probe failed'], + ['open-denied', 'not attributable — access to the process was denied'], + ['wow64-mismatch', 'not attributable on Windows — 32-bit process, 64-bit probe'], + ['reader-not-64bit', 'not attributable on Windows — 32-bit PowerShell cannot read the process'], + ['compile-failed', 'not attributable on Windows — the working-directory probe could not be built'], + ['probe-failed', 'not attributable on Windows — the working-directory probe was refused'], + ['query-failed', 'not attributable on Windows — the process could not be queried'], + ['peb-read-failed', 'not attributable on Windows — the process environment could not be read'], + ['parameters-read-failed', 'not attributable on Windows — the process environment could not be read'], + ['cwd-read-failed', 'not attributable on Windows — the working directory could not be read'], + ['empty-cwd', 'not attributable — the process recorded an empty working directory'], +]); + +const cwdDetail = (reason) => CWD_REASONS.get(reason) + ?? `not attributable — working directory unavailable (${reason})`; + +function projectOf(entry) { + if (!entry.cwd) return unknown(cwdDetail(entry.cwdReason ?? 'cwd-unavailable')); + const identity = resolveProjectIdentity(entry.cwd); + return measured({ path: entry.cwd, label: identity.label, key: identity.key }); +} + +function machineFacts(osImpl) { + let cores; + try { cores = osImpl.cpus()?.length ?? 0; } catch { cores = 0; } + return { + // The denominator every RSS figure in the System area is read against. + physicalMemoryBytes: measured(osImpl.totalmem()), + freeMemoryBytes: measured(osImpl.freemem()), + // A container with no readable CPU topology reports zero cores — an + // unmeasured quantity, not a machine with no CPUs. + cpuCount: cores > 0 ? measured(cores) : unknown('the platform reported no CPU topology'), + }; +} + +/** + * The daemon census: how many ruflo daemons are alive, how old the oldest is + * against its TTL, and what the launch budget is doing. + * + * The budget is deliberately `unknown`. `ruflo daemon budget` exists as a CLI + * but the kit has no local file or API to read it from, and synthesizing + * "$0 / idle" from the absence of evidence is precisely the fabrication + * invariant #2 forbids. When a readable source exists, this is the one field + * that changes. + */ +async function daemonCensus({ listDaemonsImpl, cwd, ttlSecs }) { + const noBudgetSource = 'ruflo exposes no local budget state this collector can read'; + try { + const daemons = await listDaemonsImpl({ cwd }); + const ages = daemons.map((entry) => entry.ageSecs).filter((age) => Number.isFinite(age)); + return { + count: measured(daemons.length), + staleCount: measured(staleDaemons(daemons, ttlSecs).length), + ttlSecs, + oldestAgeSecs: ages.length ? measured(Math.max(...ages)) : unknown(daemons.length + ? 'no running daemon recorded a start time' + : 'no daemons are running'), + budget: unknown(noBudgetSource), + entries: daemons.map((entry) => ({ + pid: entry.pid, + workspace: entry.workspace, + workspaceExists: entry.workspaceExists, + ageSecs: Number.isFinite(entry.ageSecs) + ? measured(entry.ageSecs) + : unknown('this daemon recorded no start time'), + })), + }; + } catch (error) { + const reason = `daemon discovery failed (${error?.code ?? error?.name ?? 'error'})`; + return { + count: unknown(reason), + staleCount: unknown(reason), + ttlSecs, + oldestAgeSecs: unknown(reason), + budget: unknown(noBudgetSource), + entries: [], + }; + } +} + +/** + * Collect the RuntimeCensus. Never throws: a failed process survey degrades the + * process section to unknown-with-reason while the daemon census and the + * machine denominators still render, because those are independent facts and + * losing one must not blank the view. + * + * Every collaborator is injectable, mirroring how the process survey injects + * its command runner — the win32 path is exercised by feeding fixture output + * through `surveyImpl`, not by running Windows. + * + * @param {{ + * surveyImpl?: typeof surveyHostProcesses, + * listDaemonsImpl?: typeof listDaemons, + * osImpl?: typeof os, + * platform?: NodeJS.Platform, + * cwd?: string, + * ttlSecs?: number, + * now?: number, + * }} [options] + */ +export async function collectRuntimeCensus({ + surveyImpl = surveyHostProcesses, + listDaemonsImpl = listDaemons, + osImpl = os, + platform = process.platform, + cwd = process.cwd(), + ttlSecs = Number(process.env.RUFLO_DAEMON_TTL_SECS ?? 43200), + now = Date.now(), +} = {}) { + const observedAt = new Date(now).toISOString(); + const machine = machineFacts(osImpl); + const daemons = await daemonCensus({ listDaemonsImpl, cwd, ttlSecs }); + + let survey = null; + let failure = null; + try { + survey = await surveyImpl({ platform, now }); + } catch (error) { + failure = error?.code ?? 'ERR_RUNTIME_PROCESS_SURVEY'; + } + + if (!survey) { + const reason = `the process survey could not run (${failure})`; + return { + observedAt, + platform, + ephemeral: true, + processes: unknown(reason), + childProcessCount: unknown(reason), + totals: { + processCount: unknown(reason), + rssBytes: unknown(reason), + cpuPercent: unknown(reason), + }, + daemons, + machine, + }; + } + + const rows = survey.processes.map((entry) => ({ + host: entry.host, + pid: entry.pid, + startedAt: entry.startedAt, + // Kept alongside `project` so a renderer or a debug log can key on the + // machine token while the user reads the sentence. + cwdReason: entry.cwd ? null : (entry.cwdReason ?? 'cwd-unavailable'), + uptimeMs: Number.isFinite(entry.uptimeMs) + ? measured(entry.uptimeMs) + : unknown('the process reported no usable start time'), + cpuPercent: Number.isFinite(entry.cpuPercent) + ? measured(entry.cpuPercent) + : unknown('the platform reported no CPU time for this process'), + rssBytes: Number.isFinite(entry.rssBytes) + ? measured(entry.rssBytes) + : unknown('the platform reported no resident set size for this process'), + project: projectOf(entry), + })); + + return { + observedAt: survey.observedAt ?? observedAt, + platform: survey.platform ?? platform, + ephemeral: true, + // `processes.value` is the row array, not a number — the Measurement + // wrapper is what states "the survey ran" as distinct from "no processes". + processes: measured(rows), + childProcessCount: Number.isFinite(survey.childProcessCount) + ? measured(survey.childProcessCount) + : unknown('the survey returned no process tree'), + totals: { + processCount: measured(rows.length), + // sumMeasurements marks a total `partial` when a row could not report, + // so the combined figure stays an honest lower bound instead of silently + // treating an unmeasured process as consuming nothing. + rssBytes: sumMeasurements(rows.map((row) => row.rssBytes)), + cpuPercent: sumMeasurements(rows.map((row) => row.cpuPercent)), + }, + daemons, + machine, + }; +} diff --git a/src/lib/footprint/snapshot.mjs b/src/lib/footprint/snapshot.mjs new file mode 100644 index 0000000..43825f4 --- /dev/null +++ b/src/lib/footprint/snapshot.mjs @@ -0,0 +1,250 @@ +// footprint/snapshot.mjs — persistence for the deep-tier FootprintSnapshot. +// +// ONE file, ~/.config/agentic-kit/footprint-snapshot.json, written through the +// kit's existing backup-first atomic helper. This is the machine-footprint +// context's SOLE write (ADR-0025 §6, machine-footprint invariant 4): everything +// else in the domain is read-only measurement. +// +// Two rules shape every function here: +// * Absence is not zero. A missing, unreadable, unparseable, or +// wrong-schema file degrades to "never measured" WITH a reason +// (ADR-0023 / invariant 2). No caller can ever receive a fabricated 0 from +// this module, because a failed read carries `sections: null` — there is +// no numeric field to misread. +// * The runtime census is never persisted (invariant 5). It is structurally +// impossible to write one here: `writeSnapshot` serializes only the four +// section keys in SNAPSHOT_SECTIONS, so a caller that hands over a census +// by mistake silently drops it rather than replaying a stale process table +// as liveness. +import fs from 'node:fs'; +import path from 'node:path'; +import { configDir } from '../paths.mjs'; +import { writeFileWithBackup } from '../file-write.mjs'; +import { CARRIED_FORWARD, MEASURED } from './walk.mjs'; + +/** Bump when a section's persisted SHAPE changes incompatibly. A snapshot + * written by a different version is not migrated and not guessed at — it + * reads as never-measured with an explicit reason, and the next deep scan + * replaces it. */ +export const SNAPSHOT_SCHEMA_VERSION = 1; + +/** The four deep-tier sections, in render order. This list is also the write + * filter — see the header note on the runtime census. */ +export const SNAPSHOT_SECTIONS = Object.freeze(['install', 'storage', 'catalog', 'projects']); + +/** How old a deep scan gets before the UI nudges for a rescan. Deliberately a + * nudge and not a trigger: ADR-0025's freshness policy is manual-rescan-only, + * so nothing in this module ever starts a scan on its own. */ +export const SNAPSHOT_STALE_AFTER_MS = 7 * 86_400_000; + +export function snapshotPath() { + return path.join(configDir(), 'footprint-snapshot.json'); +} + +/** + * @typedef {{ + * present: boolean, asOf: number|null, writtenAt: string|null, + * schemaVersion: number|null, completeness: object|null, + * sections: object|null, reason: string|null, file: string, + * }} PersistedSnapshot + */ + +/** The one honest shape for "there is no deep measurement to show". */ +function neverMeasured(file, reason) { + return { + present: false, + asOf: null, + writtenAt: null, + schemaVersion: null, + completeness: null, + sections: null, + reason, + file, + }; +} + +/** + * Read the persisted deep snapshot. + * + * Never throws and never returns zeros: every failure path lands on + * neverMeasured() with the reason that produced it, so the panel can say + * "not measured yet — ENOENT" instead of painting an empty machine. + * + * @param {{ file?: string, fsImpl?: typeof fs }} [options] + * @returns {PersistedSnapshot} + */ +export function readSnapshot({ file = snapshotPath(), fsImpl = fs } = {}) { + let raw; + try { + raw = fsImpl.readFileSync(file, 'utf8'); + } catch (error) { + return neverMeasured(file, error?.code === 'ENOENT' + ? 'no deep scan has been run on this machine' + : `snapshot unreadable: ${error?.code || 'io'}`); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + return neverMeasured(file, 'snapshot file is not valid JSON'); + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return neverMeasured(file, 'snapshot file is not an object'); + } + if (parsed.schemaVersion !== SNAPSHOT_SCHEMA_VERSION) { + return neverMeasured(file, + `snapshot schema ${String(parsed.schemaVersion)} is not readable by this build`); + } + // A snapshot without a usable asOf cannot honour invariant 3 (every deep + // figure carries the moment it was measured), so it is not usable at all. + if (!Number.isFinite(parsed.asOf)) { + return neverMeasured(file, 'snapshot carries no measurement time'); + } + const sections = parsed.sections; + if (!sections || typeof sections !== 'object' || Array.isArray(sections)) { + return neverMeasured(file, 'snapshot carries no sections'); + } + + return { + present: true, + asOf: parsed.asOf, + writtenAt: typeof parsed.writtenAt === 'string' ? parsed.writtenAt : null, + schemaVersion: parsed.schemaVersion, + completeness: parsed.completeness ?? null, + // Missing sections stay missing (undefined), never {}: an absent section is + // "this scan did not produce one", which the reader must be able to see. + sections: Object.fromEntries( + SNAPSHOT_SECTIONS + .filter((key) => sections[key] != null) + .map((key) => [key, sections[key]]), + ), + reason: null, + file, + }; +} + +/** + * Per-section completeness, derived from what the collectors themselves + * reported. Kept beside the data (rather than recomputed on read) so a + * carried-forward figure can still say "this section was partial when + * measured" months later. + * + * @param {object} sections + * @returns {{ complete: boolean, sections: Record, missing: string[] }} + */ +export function summarizeCompleteness(sections = {}) { + const per = {}; + const missing = []; + for (const key of SNAPSHOT_SECTIONS) { + const section = sections?.[key]; + if (section == null) { + missing.push(key); + per[key] = { measured: false, complete: false, degraded: [], truncated: [] }; + continue; + } + per[key] = { + measured: true, + complete: section.complete !== false, + degraded: Array.isArray(section.degraded) ? section.degraded : [], + truncated: Array.isArray(section.truncated) ? section.truncated + : (section.truncated ? [key] : []), + }; + } + return { + complete: missing.length === 0 && Object.values(per).every((s) => s.complete), + sections: per, + missing, + }; +} + +/** + * Persist a deep-scan result. + * + * Fail-soft by contract: a snapshot that cannot be written is a degraded + * *convenience* (the next open re-scans), never a failed scan — so this + * returns an outcome instead of throwing, and the caller keeps the in-memory + * result it just measured. + * + * @param {object} sections the four deep sections; anything else is dropped + * @param {{ file?: string, fsImpl?: typeof fs, now?: number, asOf?: number }} [options] + * @returns {{ ok: boolean, file: string, asOf: number, error: string|null }} + */ +export function writeSnapshot(sections, { + file = snapshotPath(), fsImpl = fs, now = Date.now(), asOf = null, +} = {}) { + // Prefer the collectors' own asOf over wall-clock-at-write: the figures were + // measured when the scan started, not when the file landed. + const measuredAt = Number.isFinite(asOf) ? asOf + : SNAPSHOT_SECTIONS.map((key) => sections?.[key]?.asOf).find(Number.isFinite) ?? now; + const persisted = Object.fromEntries( + SNAPSHOT_SECTIONS + .filter((key) => sections?.[key] != null) + .map((key) => [key, sections[key]]), + ); + const body = { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + asOf: measuredAt, + writtenAt: new Date(now).toISOString(), + completeness: summarizeCompleteness(persisted), + sections: persisted, + }; + try { + fsImpl.mkdirSync(path.dirname(file), { recursive: true }); + writeFileWithBackup(file, `${JSON.stringify(body)}\n`, { fsImpl }); + return { ok: true, file, asOf: measuredAt, error: null }; + } catch (error) { + return { ok: false, file, asOf: measuredAt, error: String(error?.code || error?.message || error) }; + } +} + +/** Does this look like a walk.mjs Measurement? Checked structurally rather than + * by class because the value crossed JSON on the way in. */ +function isMeasurement(value) { + return !!value && typeof value === 'object' && !Array.isArray(value) + && typeof value.status === 'string' + && 'value' in value && 'reason' in value && 'asOf' in value && 'partial' in value; +} + +/** + * Re-stamp a persisted section tree as carried-forward. + * + * Invariant 3: data from a previous scan is never presented as current. A + * `measured` figure read back off disk becomes `carried-forward` with THAT + * scan's asOf, so a renderer cannot mistake it for something this request + * measured. `unknown` stays unknown (a failed measurement does not improve by + * being persisted) and an already carried-forward figure is left alone. + * + * Pure and recursive over plain JSON — the input has no cycles by construction. + * + * @param {*} value + * @param {number} asOf the snapshot's own measurement time + */ +export function carryForward(value, asOf) { + if (Array.isArray(value)) return value.map((item) => carryForward(item, asOf)); + if (!value || typeof value !== 'object') return value; + if (isMeasurement(value)) { + if (value.status !== MEASURED) return value; + return { ...value, status: CARRIED_FORWARD, asOf }; + } + const out = {}; + for (const [key, item] of Object.entries(value)) out[key] = carryForward(item, asOf); + return out; +} + +/** + * Freshness facts for a read snapshot. `stale` drives the visible rescan nudge + * and nothing else — no code path in this domain auto-scans on it. + * + * @param {PersistedSnapshot} snapshot + * @param {{ now?: number, staleAfterMs?: number }} [options] + */ +export function snapshotFreshness(snapshot, { + now = Date.now(), staleAfterMs = SNAPSHOT_STALE_AFTER_MS, +} = {}) { + if (!snapshot?.present || !Number.isFinite(snapshot.asOf)) { + return { measured: false, asOf: null, ageMs: null, stale: false, staleAfterMs }; + } + const ageMs = Math.max(0, now - snapshot.asOf); + return { measured: true, asOf: snapshot.asOf, ageMs, stale: ageMs > staleAfterMs, staleAfterMs }; +} diff --git a/src/lib/footprint/storage.mjs b/src/lib/footprint/storage.mjs new file mode 100644 index 0000000..7166a18 --- /dev/null +++ b/src/lib/footprint/storage.mjs @@ -0,0 +1,663 @@ +// Storage breakdown — the category → host → project → session tree, the derived +// views over the same walk (trailing-30d growth, top-N giants), and the advisory +// reclaimable rows (ADR-0025 §4, docs/ddd/machine-footprint.md "Storage +// breakdown"). +// +// This module is ADVISORY ONLY by invariant 4: there is no delete, prune, or +// cleanup verb here, and none may be added. `ReclaimableCandidate` rows carry a +// path, a size, and a rationale — a `cleanupHint` names the CLI that already +// owns the removal, and that string is documentation, not a command this module +// runs. npx.mjs's pruneNpxStale is deliberately NOT imported; only its +// read-only scanNpxStale is. +// +// Metadata only (invariant 1). Every figure comes from dirents, lstat sizes and +// mtimes. A transcript's contents are never opened — which is also why codex +// transcripts have no project attribution here: codex rollout PATHS are dated, +// not project-scoped, and the project lives inside the file. That reads as an +// honest "unattributable", never as a guess and never as a zero. +// +// One deliberate exception, narrow and documented: orphaned-worktree detection +// reads `/.git/worktrees//gitdir`, which holds a single filesystem +// PATH. That is the same class of datum as `.git/config`'s remote URL, which +// ADR-0025 §7 already sanctions, and it is the only way an orphaned worktree +// can be identified at all. Bounded to 4 KB, parsed as a path and nothing else, +// and skipped entirely when `detectWorktrees` is false. +import fs from 'node:fs'; +import path from 'node:path'; +import { home, claudeDir, codexDir, configDir } from '../paths.mjs'; +import { defaultOpencodeDbPath } from '../usage-opencode.mjs'; +import { scanNpxStale } from '../npx.mjs'; +import { npxEnvNodes } from './install.mjs'; +import { + walkTree, rootMeasurements, measured, unknown, sumMeasurements, statNode, +} from './walk.mjs'; + +export const STORAGE_CATEGORIES = Object.freeze([ + 'transcripts', 'ledgers-and-logs', 'learning-stores', 'kit-caches', +]); + +export const STORAGE_DEFAULTS = Object.freeze({ + growthDays: 30, + topN: 10, + // A codex sessions root holds one file per session and years of them; the + // tree needs a ceiling that keeps the payload renderable. The remainder is + // folded into one aggregate child so the parent's total still adds up. + maxChildren: 200, + transcriptAgeDays: 180, + npxEnvIdleDays: 90, + worktreeIdleDays: 90, + maxWorktreeWalks: 32, + samplePaths: 5, +}); + +/** Host ledger and log files that are named by generation + * (`state_5.sqlite`, `logs_2.sqlite`, plus their -wal/-shm siblings). Matching + * the family rather than one name is deliberate: codex bumps the generation + * suffix on schema changes and leaves the old file behind. */ +const CODEX_LEDGER_RE = /^[a-z_]+_\d+\.sqlite(?:-wal|-shm)?$/; +const OPENCODE_STORE_RE = /^opencode\.db(?:-wal|-shm)?$/; +const flatDir = () => true; + +/** + * @typedef {{ + * id: string, category: string, host: string, label: string, path: string, + * layout: 'claude-projects'|'flat-sessions'|'tree', + * project?: string, projectPath?: string, + * acceptFile?: (name: string) => boolean, + * skipDir?: (dir: string, name: string, depth: number) => boolean, + * }} StorageRoot + */ + +/** + * The known roots this domain measures, one descriptor per node. + * + * `layout` decides how leaves are derived: + * claude-projects `//.jsonl` — project from the + * directory name, session from the file. + * flat-sessions session leaves with no path-derivable project (codex + * rollouts, the opencode store). + * tree no leaves; the root is the node, files aggregate into it. + * + * `projects`, when supplied, adds the per-project learning stores. Passing null + * (the default) means "no project catalog was supplied" — that category then + * reports unknown rather than a fabricated zero; passing `[]` means a catalog + * was supplied and was genuinely empty, which is a measured zero. + * + * @returns {StorageRoot[]} + */ +export function defaultStorageRoots({ env = process.env, projects = null } = {}) { + const stateRoot = env.XDG_STATE_HOME || path.join(home, '.local', 'state'); + const opencodeData = path.dirname(defaultOpencodeDbPath()); + const claude = (name) => path.join(claudeDir(), name); + const codex = (name) => path.join(codexDir(), name); + + /** @type {StorageRoot[]} */ + const roots = [ + { + id: 'claude-transcripts', category: 'transcripts', host: 'claude', + label: 'session transcripts', path: claude('projects'), layout: 'claude-projects', + }, + { + id: 'codex-transcripts', category: 'transcripts', host: 'codex', + label: 'session rollouts', path: codex('sessions'), layout: 'flat-sessions', + }, + { + id: 'opencode-store', category: 'transcripts', host: 'opencode', + label: 'session store', path: opencodeData, layout: 'flat-sessions', + acceptFile: (name) => OPENCODE_STORE_RE.test(name), skipDir: flatDir, + }, + { + id: 'codex-ledgers', category: 'ledgers-and-logs', host: 'codex', + label: 'thread ledgers', path: codexDir(), layout: 'tree', + acceptFile: (name) => CODEX_LEDGER_RE.test(name), skipDir: flatDir, + }, + { + id: 'codex-history', category: 'ledgers-and-logs', host: 'codex', + label: 'history.jsonl', path: codex('history.jsonl'), layout: 'tree', + }, + { + id: 'codex-shell-snapshots', category: 'ledgers-and-logs', host: 'codex', + label: 'shell snapshots', path: codex('shell_snapshots'), layout: 'tree', + }, + { + id: 'claude-logs', category: 'ledgers-and-logs', host: 'claude', + label: 'logs', path: claude('logs'), layout: 'tree', + }, + { + id: 'claude-debug', category: 'ledgers-and-logs', host: 'claude', + label: 'debug', path: claude('debug'), layout: 'tree', + }, + { + id: 'claude-telemetry', category: 'ledgers-and-logs', host: 'claude', + label: 'telemetry', path: claude('telemetry'), layout: 'tree', + }, + { + id: 'claude-statsig', category: 'ledgers-and-logs', host: 'claude', + label: 'statsig', path: claude('statsig'), layout: 'tree', + }, + { + id: 'claude-shell-snapshots', category: 'ledgers-and-logs', host: 'claude', + label: 'shell snapshots', path: claude('shell-snapshots'), layout: 'tree', + }, + { + id: 'claude-history', category: 'ledgers-and-logs', host: 'claude', + label: 'history.jsonl', path: claude('history.jsonl'), layout: 'tree', + }, + { + id: 'opencode-logs', category: 'ledgers-and-logs', host: 'opencode', + label: 'logs', path: path.join(opencodeData, 'log'), layout: 'tree', + }, + { + id: 'ak-runtime-debug', category: 'ledgers-and-logs', host: 'agentic-kit', + label: 'runtime-debug.log', + path: path.join(stateRoot, 'agentic-kit', 'runtime-debug.log'), layout: 'tree', + }, + { + id: 'ak-config', category: 'kit-caches', host: 'agentic-kit', + label: 'config, indexes & snapshots', path: configDir(), layout: 'tree', + }, + { + id: 'claude-cache', category: 'kit-caches', host: 'claude', + label: 'cache', path: claude('cache'), layout: 'tree', + }, + { + id: 'claude-image-cache', category: 'kit-caches', host: 'claude', + label: 'image cache', path: claude('image-cache'), layout: 'tree', + }, + { + id: 'claude-paste-cache', category: 'kit-caches', host: 'claude', + label: 'paste cache', path: claude('paste-cache'), layout: 'tree', + }, + { + id: 'codex-cache', category: 'kit-caches', host: 'codex', + label: 'cache', path: codex('cache'), layout: 'tree', + }, + ]; + + for (const project of projects ?? []) { + const label = path.basename(project); + for (const [dir, what] of [ + ['.claude-flow', 'ruflo learning state'], + ['.agentic-qe', 'agentic-qe state'], + ['.swarm', 'swarm memory'], + ]) { + roots.push({ + id: `learning:${project}:${dir}`, + category: 'learning-stores', + host: 'project', + project: label, + projectPath: project, + label: what, + path: path.join(project, dir), + layout: 'tree', + }); + } + } + return roots; +} + +/** YYYY-MM-DD in LOCAL time — the same convention usage-index's localDay uses, + * so a growth day and a usage day mean the same calendar day. */ +export function localDay(ms) { + const d = new Date(ms); + const p = (n) => String(n).padStart(2, '0'); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`; +} + +function newNode({ key, kind, label, path: nodePath = null, host = null, attribution = null }) { + return { + key, kind, label, path: nodePath, host, attribution, + bytes: 0, files: 0, newestMtimeMs: null, presence: 'present', + children: new Map(), + }; +} + +function childOf(children, key, spec) { + if (!children.has(key)) children.set(key, newNode({ key, ...spec })); + return children.get(key); +} + +function bump(node, bytes, mtimeMs) { + node.bytes += bytes; + node.files += 1; + if (node.newestMtimeMs === null || mtimeMs > node.newestMtimeMs) node.newestMtimeMs = mtimeMs; +} + +/** Depth-first finalize: Map children → sorted array, capped, remainder folded + * into one aggregate node so a capped parent's total still adds up. */ +function finalizeNode(node, { asOf, maxChildren }) { + const kids = [...node.children.values()] + .map((child) => finalizeNode(child, { asOf, maxChildren })) + .sort((a, b) => (b.bytes.value ?? 0) - (a.bytes.value ?? 0)); + let children = kids; + if (kids.length > maxChildren) { + const kept = kids.slice(0, maxChildren); + const rest = kids.slice(maxChildren); + const restBytes = rest.reduce((acc, c) => acc + (c.bytes.value ?? 0), 0); + const restFiles = rest.reduce((acc, c) => acc + (c.files.value ?? 0), 0); + children = [...kept, { + key: `${node.key}::others`, + kind: 'aggregate', + label: `${rest.length} more`, + path: null, + host: node.host, + attribution: null, + presence: 'present', + bytes: measured(restBytes, { asOf }), + files: measured(restFiles, { asOf }), + newestMtimeMs: null, + children: [], + }]; + } + return { + key: node.key, + kind: node.kind, + label: node.label, + path: node.path, + host: node.host, + attribution: node.attribution, + presence: node.presence, + bytes: node.bytesMeasurement ?? measured(node.bytes, { asOf, partial: node.partial === true }), + files: node.filesMeasurement ?? measured(node.files, { asOf, partial: node.partial === true }), + newestMtimeMs: node.newestMtimeMs, + children, + }; +} + +/** Where a file sits in the project/session part of the tree, from its PATH + * alone. `attribution` states how the project was determined so the UI can say + * "unattributable" rather than showing an empty cell. */ +function leafKeysFor(root, file) { + const rel = path.relative(root.path, file); + const parts = rel.split(path.sep).filter(Boolean); + const session = parts.length ? parts[parts.length - 1] : path.basename(file); + if (root.layout === 'claude-projects' && parts.length > 1) { + return { project: parts[0], attribution: 'path', session }; + } + if (root.layout === 'flat-sessions' || root.layout === 'claude-projects') { + return { project: null, attribution: 'none', session }; + } + return { project: root.project ?? null, attribution: root.project ? 'catalog' : 'root', session: null }; +} + +/** + * The storage section of a FootprintSnapshot. + * + * @param {{ + * projects?: string[]|null, now?: () => number, walk?: typeof walkTree, + * roots?: StorageRoot[]|null, limits?: object, growthDays?: number, topN?: number, + * maxChildren?: number, reclaim?: object, detectWorktrees?: boolean, + * fsImpl?: typeof fs, + * }} [options] + */ +export function collectStorage({ + projects = null, + now = Date.now, + walk = walkTree, + roots = null, + limits = {}, + growthDays = STORAGE_DEFAULTS.growthDays, + topN = STORAGE_DEFAULTS.topN, + maxChildren = STORAGE_DEFAULTS.maxChildren, + reclaim = {}, + detectWorktrees = true, + fsImpl = fs, +} = {}) { + const asOf = now(); + const opts = { ...STORAGE_DEFAULTS, ...reclaim }; + const rootList = roots ?? defaultStorageRoots({ projects }); + const growthCutoff = asOf - growthDays * 86_400_000; + const agedCutoff = asOf - opts.transcriptAgeDays * 86_400_000; + + const categories = new Map(); + const growth = new Map(); + const sessionLeaves = []; + const files = []; + const agedTranscripts = new Map(); + let anyDegraded = false; + + const trimTop = (list) => { + if (list.length <= topN * 8) return; + list.sort((a, b) => b.bytes - a.bytes); + list.length = topN; + }; + + for (const root of rootList) { + const category = childOf(categories, root.category, { + kind: 'category', label: root.category, + }); + const host = childOf(category.children, root.host, { + kind: 'host', label: root.host, host: root.host, + }); + + const projectKey = root.project ?? root.id; + const rootNode = root.layout === 'claude-projects' + ? host + : childOf(host.children, projectKey, { + kind: 'project', + label: root.project ?? root.label, + path: root.path, + host: root.host, + attribution: root.project ? 'catalog' : 'root', + }); + + const result = walk(root.path, { + ...limits, + fsImpl, + ...(root.acceptFile ? { acceptFile: (name) => root.acceptFile(name) } : {}), + ...(root.skipDir ? { skipDir: root.skipDir } : {}), + onFile: ({ file, name, bytes, mtimeMs }) => { + bump(category, bytes, mtimeMs); + bump(host, bytes, mtimeMs); + if (rootNode !== host) bump(rootNode, bytes, mtimeMs); + + const { project, attribution, session } = leafKeysFor(root, file); + let leafParent = rootNode; + if (root.layout === 'claude-projects') { + leafParent = childOf(host.children, project ?? '(unattributed)', { + kind: 'project', + label: project ?? 'unattributed', + path: project ? path.join(root.path, project) : root.path, + host: root.host, + attribution, + }); + bump(leafParent, bytes, mtimeMs); + } else if (root.layout === 'flat-sessions') { + rootNode.attribution = 'none'; + } + if (session) { + const leaf = childOf(leafParent.children, session, { + kind: 'session', label: session, path: file, host: root.host, attribution, + }); + bump(leaf, bytes, mtimeMs); + sessionLeaves.push({ + session, host: root.host, category: root.category, + project: project ?? null, attribution, path: file, bytes, mtimeMs, + }); + trimTop(sessionLeaves); + } + + if (mtimeMs >= growthCutoff) { + if (!growth.has(root.host)) growth.set(root.host, new Map()); + const days = growth.get(root.host); + const day = localDay(mtimeMs); + const cell = days.get(day) ?? { bytes: 0, files: 0 }; + cell.bytes += bytes; + cell.files += 1; + days.set(day, cell); + } + + files.push({ path: file, name, host: root.host, category: root.category, bytes, mtimeMs }); + trimTop(files); + + if (root.category === 'transcripts' && mtimeMs < agedCutoff) { + const acc = agedTranscripts.get(root.host) + ?? { host: root.host, root: root.path, files: 0, bytes: 0, oldestMtimeMs: null, samples: [] }; + acc.files += 1; + acc.bytes += bytes; + if (acc.oldestMtimeMs === null || mtimeMs < acc.oldestMtimeMs) acc.oldestMtimeMs = mtimeMs; + if (acc.samples.length < opts.samplePaths) acc.samples.push(file); + agedTranscripts.set(root.host, acc); + } + }, + }); + + const { presence } = rootMeasurements(result, { asOf }); + if (rootNode !== host) rootNode.presence = presence; + if (result.complete || presence === 'absent') continue; + anyDegraded = true; + // Invariant 6: this root degrades, its siblings under the same host keep + // their measured figures. The ancestors stay measured but become `partial` + // — a floor — because a sum that silently omits an unknown child would + // read as a total, which is the zero-for-unknown failure in disguise. + if (rootNode !== host) { + if (result.status === 'unknown') { + rootNode.bytesMeasurement = unknown(result.reason); + rootNode.filesMeasurement = unknown(result.reason); + } else { + rootNode.partial = true; + } + } + host.partial = true; + category.partial = true; + } + + // Every category always appears, so a missing slice is never mistaken for a + // rendering gap. A category with no roots is a measured zero — EXCEPT + // learning-stores with no project catalog supplied, whose emptiness is + // ambiguous: "we were given nowhere to look" is not "there is nothing there". + for (const id of STORAGE_CATEGORIES) { + if (categories.has(id)) continue; + const node = newNode({ key: id, kind: 'category', label: id }); + if (id === 'learning-stores' && projects === null) { + node.presence = 'unknown'; + node.bytesMeasurement = unknown('no project catalog supplied'); + node.filesMeasurement = unknown('no project catalog supplied'); + } + categories.set(id, node); + } + + const tree = [...categories.values()] + .map((node) => finalizeNode(node, { asOf, maxChildren })) + .sort((a, b) => (b.bytes.value ?? 0) - (a.bytes.value ?? 0)); + + files.sort((a, b) => b.bytes - a.bytes); + sessionLeaves.sort((a, b) => b.bytes - a.bytes); + + return { + asOf, + categories: tree, + totals: { + bytes: sumMeasurements(tree.map((c) => c.bytes), { asOf }), + files: sumMeasurements(tree.map((c) => c.files), { asOf }), + }, + growth: buildGrowth(growth, { asOf, growthDays }), + topSessions: sessionLeaves.slice(0, topN), + topFiles: files.slice(0, topN), + reclaimables: collectReclaimables({ + asOf, agedTranscripts, projects, opts, walk, limits, detectWorktrees, fsImpl, + }), + complete: !anyDegraded, + }; +} + +/** Trailing-window growth per host, from mtime + size only — no content read + * ever happens here. It is an APPROXIMATION and says so: a file rewritten + * today contributes its WHOLE size to today, because a single mtime cannot + * tell how many of those bytes are new. That over-counts rewritten stores + * (sqlite ledgers) and is exact for append-only transcripts. */ +export function buildGrowth(growthByHost, { asOf, growthDays }) { + const days = []; + for (let i = growthDays - 1; i >= 0; i--) days.push(localDay(asOf - i * 86_400_000)); + const hosts = [...growthByHost.entries()].map(([host, cells]) => { + const series = days.map((day) => ({ + day, + bytes: cells.get(day)?.bytes ?? 0, + files: cells.get(day)?.files ?? 0, + })); + const total = series.reduce((acc, d) => acc + d.bytes, 0); + return { + host, + days: series, + totalBytes: measured(total, { asOf }), + perDayAvgBytes: measured(Math.round(total / growthDays), { asOf }), + }; + }).sort((a, b) => (b.totalBytes.value ?? 0) - (a.totalBytes.value ?? 0)); + return { + windowDays: growthDays, + approximate: true, + basis: 'file mtime + size; a rewritten file counts its whole size on its mtime day', + hosts, + }; +} + +/** Advisory rows only — see this module's header. Nothing here removes + * anything; `cleanupHint` names the CLI that already owns the removal. */ +export function collectReclaimables({ + asOf, agedTranscripts, projects, opts, walk, limits, detectWorktrees, fsImpl, +}) { + const rows = []; + const days = (ms) => Math.floor((asOf - ms) / 86_400_000); + + for (const acc of agedTranscripts.values()) { + rows.push({ + id: `aged-transcripts:${acc.host}`, + kind: 'aged-transcripts', + label: `${acc.host} transcripts older than ${opts.transcriptAgeDays}d`, + path: acc.root, + samplePaths: acc.samples, + bytes: measured(acc.bytes, { asOf }), + files: measured(acc.files, { asOf }), + rationale: `${acc.files} file(s) untouched for ${opts.transcriptAgeDays}d or more; ` + + `oldest ${days(acc.oldestMtimeMs)}d. Historical usage reads these — removing them ` + + 'removes that history too.', + cleanupHint: null, + advisory: true, + }); + } + + rows.push(...npxReclaimables({ asOf, opts, walk, limits, fsImpl })); + if (detectWorktrees && Array.isArray(projects)) { + rows.push(...worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImpl })); + } + return rows.sort((a, b) => (b.bytes.value ?? 0) - (a.bytes.value ?? 0)); +} + +/** Stale npx cache envs. Two independent rationales, both read-only: a cached + * copy strictly older than its installed global baseline (npx.mjs's version + * verdict — the bug that kept a machine running a retired ruflo), and an env + * untouched for longer than the idle threshold. */ +export function npxReclaimables({ asOf, opts, walk, limits, fsImpl }) { + const nodes = npxEnvNodes({ walk, limits, asOf, fsImpl }); + if (nodes.presence !== 'present') return []; + let staleByVersion = new Map(); + try { + staleByVersion = new Map(scanNpxStale().map((entry) => [entry.dir, entry.stale])); + } catch { /* an unreadable cache simply yields no version verdict */ } + const idleCutoff = asOf - opts.npxEnvIdleDays * 86_400_000; + const rows = []; + for (const env of nodes.envs) { + const stale = staleByVersion.get(env.path); + const idle = env.newestMtimeMs !== null && env.newestMtimeMs < idleCutoff; + if (!stale && !idle) continue; + const why = []; + if (stale) { + why.push(`cached ${stale.map((s) => `${s.pkg}@${s.cached}`).join(', ')} ` + + `older than installed ${stale.map((s) => s.installed).join(', ')}`); + } + if (idle) why.push(`untouched for ${Math.floor((asOf - env.newestMtimeMs) / 86_400_000)}d`); + rows.push({ + id: `stale-npx-env:${env.id}`, + kind: 'stale-npx-env', + label: `npx cache env (${env.packages.join(', ') || 'unkeyed'})`, + path: env.path, + samplePaths: [], + bytes: env.bytes, + files: env.files, + rationale: `${why.join('; ')}. npx re-fetches on demand, so the cache is reproducible.`, + cleanupHint: 'ak sync prunes version-stale envs (npx.pruneNpxStale)', + advisory: true, + }); + } + return rows; +} + +/** Orphaned git worktrees, from each project's `.git/worktrees/` admin + * records. Two honest verdicts, no git invocation: + * · the checkout the record points at no longer exists → the record is dead; + * · the checkout exists but nothing in it has been touched for the idle + * window → a candidate, with its real on-disk size. + * A record whose pointer cannot be read is reported as unverifiable rather + * than assumed dead. See this module's header for why the pointer read is in + * scope. */ +export function worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImpl }) { + const rows = []; + let walks = 0; + for (const project of projects) { + const adminRoot = path.join(project, '.git', 'worktrees'); + let entries; + try { + entries = fsImpl.readdirSync(adminRoot, { withFileTypes: true }); + } catch { continue; } // no worktrees here (or unreadable): nothing to claim + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const record = path.join(adminRoot, entry.name); + const pointer = readGitdirPointer(path.join(record, 'gitdir'), fsImpl); + if (!pointer) { + rows.push({ + id: `orphaned-worktree:${record}`, + kind: 'orphaned-worktree', + label: `worktree record "${entry.name}" (unverifiable)`, + path: record, + samplePaths: [], + bytes: unknown('worktree pointer unreadable'), + files: unknown('worktree pointer unreadable'), + rationale: 'The admin record exists but its gitdir pointer could not be read, ' + + 'so whether the checkout still exists is unknown.', + cleanupHint: 'git worktree prune (verify first)', + advisory: true, + }); + continue; + } + // gitdir points at `/.git`; the checkout is its parent. + const checkout = path.dirname(pointer); + const head = statNode(checkout, { fsImpl }); + if (head.status === 'unknown') { + rows.push({ + id: `orphaned-worktree:${record}`, + kind: 'orphaned-worktree', + label: `orphaned worktree record "${entry.name}"`, + path: record, + samplePaths: [checkout], + bytes: measured(0, { asOf }), + files: measured(0, { asOf }), + rationale: `The checkout at ${checkout} no longer exists; only the administrative ` + + 'record remains.', + cleanupHint: 'git worktree prune', + advisory: true, + }); + continue; + } + if (walks >= opts.maxWorktreeWalks) continue; + walks += 1; + const result = walk(checkout, { ...limits, fsImpl }); + const { bytes, files } = rootMeasurements(result, { asOf }); + const idleMs = result.newestMtimeMs === null ? null : asOf - result.newestMtimeMs; + if (idleMs === null || idleMs < opts.worktreeIdleDays * 86_400_000) continue; + rows.push({ + id: `idle-worktree:${record}`, + kind: 'orphaned-worktree', + label: `idle worktree "${entry.name}"`, + path: checkout, + samplePaths: [record], + bytes, + files, + rationale: `Nothing in the checkout has changed for ${Math.floor(idleMs / 86_400_000)}d. ` + + 'Merge state is not checked here — confirm the branch is landed before removing.', + cleanupHint: 'git worktree remove (verify the branch is merged first)', + advisory: true, + }); + } + } + return rows; +} + +/** The single narrow non-metadata read in this module: a `gitdir` file holds + * one absolute path and nothing else. Bounded to 4 KB and validated as a path + * so a corrupt file yields null rather than a bogus candidate. */ +function readGitdirPointer(file, fsImpl) { + let fd; + try { + fd = fsImpl.openSync(file, 'r'); + const buf = Buffer.alloc(4096); + const read = fsImpl.readSync(fd, buf, 0, 4096, 0); + const value = buf.toString('utf8', 0, read).trim(); + return value && path.isAbsolute(value) ? value : null; + } catch { + return null; + } finally { + if (fd !== undefined) { + try { fsImpl.closeSync(fd); } catch { /* already gone */ } + } + } +} diff --git a/src/lib/footprint/walk.mjs b/src/lib/footprint/walk.mjs new file mode 100644 index 0000000..a31e121 --- /dev/null +++ b/src/lib/footprint/walk.mjs @@ -0,0 +1,289 @@ +// The ONE bounded directory walker every machine-footprint collector uses, and +// the Measurement vocabulary they all report in (ADR-0025, docs/ddd/machine- +// footprint.md invariants 2 and 6). No other module in this context may readdir +// on its own: the caps, the never-follow-symlinks rule, and the +// degrade-this-node-only failure mode are safety-critical, so they get exactly +// one implementation to audit rather than one per collector. +// +// Metadata only, structurally. This module has no read path for file contents — +// readdir entries and lstat results are all it can obtain — so no collector +// built on it can acquire one by accident (invariant 1). +// +// The limits are explicit, not inherited, so a caller reads what it is buying: +// +// maxDepth How far below `root` the walk descends. A global npm tree nests +// node_modules several levels; codex transcript roots are four +// deep (YYYY/MM/DD/file). 16 covers both without letting a +// pathological tree run away. Hitting it marks the walk truncated +// — the figures are then a floor, never a total. +// maxEntries Every dirent seen counts, including skipped symlinks. This is +// the wall-clock bound: a walk that exhausts it stops and says so. +// maxDegraded How many unreadable-node reasons are RETAINED. The degraded +// COUNT is always exact; only the retained sample is capped, so a +// permission-denied storm cannot balloon the payload. +// +// Symlinks are never followed and never counted. A symlinked tree's bytes +// belong to wherever they really live, counting them here would double-count +// them there, and following them is how a walker escapes its root or spins +// forever on a cycle. +import fs from 'node:fs'; +import path from 'node:path'; + +export const WALK_LIMITS = Object.freeze({ + maxDepth: 16, + maxEntries: 400_000, + maxDegraded: 64, +}); + +export const MEASURED = 'measured'; +export const CARRIED_FORWARD = 'carried-forward'; +export const UNKNOWN = 'unknown'; + +/** A real walk/stat/count produced this. `partial: true` means a cap or an + * unreadable subtree cut the measurement short, so the value is a floor. */ +export function measured(value, { asOf = null, partial = false } = {}) { + return { value, status: MEASURED, reason: null, asOf, partial }; +} + +/** A figure from a previous deep scan, presented with THAT scan's asOf — + * never as current (invariant 3). */ +export function carriedForward(value, asOf) { + return { value, status: CARRIED_FORWARD, reason: null, asOf, partial: false }; +} + +/** Never measured, or the measurement failed. `reason` is mandatory: an + * unknown without a why is exactly the failure mode ADR-0023 exists to + * prevent, and it is the ONLY correct alternative to a number — a figure the + * collector does not have is never rendered as 0 (invariant 2). */ +export function unknown(reason) { + return { + value: null, + status: UNKNOWN, + reason: String(reason || 'unmeasured'), + asOf: null, + partial: false, + }; +} + +/** Did this snapshot's own collectors produce the figure (as opposed to a + * carried-forward one)? */ +export const isMeasured = (m) => m?.status === MEASURED; +/** Is there a number at all — measured or carried forward? */ +export const hasValue = (m) => typeof m?.value === 'number' && Number.isFinite(m.value); + +/** Sum of the inputs that have values. An unknown input never contributes a 0; + * it makes the sum `partial`, i.e. an honest lower bound. An all-unknown sum + * is unknown, not 0. Summing an EMPTY list is a measured 0 — a caller that + * does not know whether there were inputs must pass an explicit unknown() + * rather than an empty array. */ +export function sumMeasurements(list, { asOf = null } = {}) { + const items = Array.isArray(list) ? list : []; + if (!items.length) return measured(0, { asOf }); + const usable = items.filter(hasValue); + if (!usable.length) return unknown('every input unmeasured'); + const total = usable.reduce((acc, m) => acc + m.value, 0); + const partial = usable.length < items.length || usable.some((m) => m.partial); + return measured(total, { asOf, partial }); +} + +/** lstat one known path. Used for the individually-known files the cheap tier + * reads (ledgers, tee files, index caches) where a walk would be overkill. + * Never follows a symlink — a symlinked known-file reports kind 'symlink' and + * no bytes rather than silently measuring its target. */ +export function statNode(target, { fsImpl = fs } = {}) { + try { + const st = fsImpl.lstatSync(target); + const kind = st.isFile() ? 'file' + : st.isDirectory() ? 'dir' + : st.isSymbolicLink() ? 'symlink' : 'other'; + return { + path: target, + status: MEASURED, + reason: null, + kind, + bytes: kind === 'file' ? st.size : null, + mtimeMs: st.mtimeMs, + }; + } catch (err) { + return { + path: target, + status: UNKNOWN, + reason: err?.code || 'io', + kind: null, + bytes: null, + mtimeMs: null, + }; + } +} + +/** + * Bounded, symlink-free tree walk. Returns bytes, file count, and newest mtime + * for `root`, plus the provenance a caller needs to render the figure honestly. + * + * Failure semantics, per invariant 6: an unreadable subtree degrades THAT node + * — it is recorded in `degraded` and the walk continues with its siblings. Only + * an unreadable ROOT makes the whole result unknown. `complete` is the single + * flag a caller checks: false means a cap fired or something was unreadable, so + * `bytes`/`files` are floors. + * + * @param {string} root + * @param {{ + * maxDepth?: number, maxEntries?: number, maxDegraded?: number, + * skipDir?: ((dir: string, name: string, depth: number) => boolean) | null, + * acceptFile?: ((name: string, file: string, depth: number) => boolean) | null, + * onFile?: ((entry: { file: string, name: string, bytes: number, + * mtimeMs: number, depth: number }) => void) | null, + * fsImpl?: typeof fs, + * }} [options] `skipDir` prunes a subtree deliberately (it does NOT mark the + * walk truncated — an intentional scope is not a failed measurement). + * `acceptFile` filters what is COUNTED and what reaches `onFile`; rejected + * files still consume the entry budget. `onFile` exceptions propagate: a + * collector bug must surface, not be swallowed as an unreadable subtree. + */ +export function walkTree(root, options = {}) { + const { + maxDepth = WALK_LIMITS.maxDepth, + maxEntries = WALK_LIMITS.maxEntries, + maxDegraded = WALK_LIMITS.maxDegraded, + skipDir = null, + acceptFile = null, + onFile = null, + fsImpl = fs, + } = options; + + const result = { + root, + status: MEASURED, + reason: null, + bytes: 0, + files: 0, + dirs: 0, + newestMtimeMs: null, + entriesSeen: 0, + symlinksSkipped: 0, + truncated: false, + truncatedBy: null, + degradedCount: 0, + degraded: [], + complete: true, + }; + + const degrade = (target, reason) => { + result.degradedCount += 1; + result.complete = false; + if (result.degraded.length < maxDegraded) result.degraded.push({ path: target, reason }); + }; + const truncate = (by) => { + if (!result.truncated) { result.truncated = true; result.truncatedBy = by; } + result.complete = false; + }; + const countFile = (file, name, st, depth) => { + result.files += 1; + result.bytes += st.size; + if (result.newestMtimeMs === null || st.mtimeMs > result.newestMtimeMs) { + result.newestMtimeMs = st.mtimeMs; + } + if (onFile) onFile({ file, name, bytes: st.size, mtimeMs: st.mtimeMs, depth }); + }; + + // A root that is itself a file is a legitimate node (opencode's single store, + // a tee log). A root that is a symlink is refused rather than followed. + const head = statNode(root, { fsImpl }); + if (head.status === UNKNOWN) { + return { ...result, status: UNKNOWN, reason: head.reason, bytes: null, files: null, complete: false }; + } + if (head.kind === 'symlink') { + return { ...result, status: UNKNOWN, reason: 'symlink (never followed)', bytes: null, files: null, complete: false }; + } + if (head.kind !== 'dir') { + if (!acceptFile || acceptFile(path.basename(root), root, 0)) { + countFile(root, path.basename(root), { size: head.bytes ?? 0, mtimeMs: head.mtimeMs }, 0); + } + return result; + } + + const stack = [{ dir: root, depth: 0 }]; + while (stack.length) { + if (result.entriesSeen >= maxEntries) { truncate('entries'); break; } + const { dir, depth } = stack.pop(); + let entries; + try { + entries = fsImpl.readdirSync(dir, { withFileTypes: true }); + } catch (err) { + const code = err?.code || 'io'; + // The root itself being unlistable is not a degraded subtree — there is + // no sibling evidence to keep, so the whole node is unknown. Reporting it + // as a "partial 0" would be exactly the unknown-rendered-as-zero failure + // invariant 2 forbids: an EACCES directory is not an empty one. + if (dir === root) { + return { ...result, status: UNKNOWN, reason: code, bytes: null, files: null, complete: false }; + } + degrade(dir, code); + continue; + } + result.dirs += 1; + for (const entry of entries) { + if (result.entriesSeen >= maxEntries) { truncate('entries'); break; } + result.entriesSeen += 1; + const file = path.join(dir, entry.name); + if (entry.isSymbolicLink()) { result.symlinksSkipped += 1; continue; } + if (entry.isDirectory()) { + if (depth + 1 > maxDepth) { truncate('depth'); continue; } + if (skipDir && skipDir(file, entry.name, depth + 1)) continue; + stack.push({ dir: file, depth: depth + 1 }); + continue; + } + // Sockets, fifos and device nodes hold no bytes this domain owns. + if (!entry.isFile()) continue; + if (acceptFile && !acceptFile(entry.name, file, depth + 1)) continue; + let st; + try { + st = fsImpl.lstatSync(file); + } catch (err) { + degrade(file, err?.code || 'io'); + continue; + } + countFile(file, entry.name, st, depth + 1); + } + } + return result; +} + +/** A walk's byte/file figures as Measurements. An unreadable root yields + * unknown-with-reason; a truncated or partially degraded walk yields + * `partial: true` so the UI can render "at least", never a bare total. */ +export function walkMeasurements(walk, { asOf = null } = {}) { + if (!walk || walk.status === UNKNOWN) { + const reason = walk?.reason || 'not measured'; + return { bytes: unknown(reason), files: unknown(reason) }; + } + const partial = !walk.complete; + return { + bytes: measured(walk.bytes, { asOf, partial }), + files: measured(walk.files, { asOf, partial }), + }; +} + +/** ENOENT on a root is an ABSENCE, not a failed measurement: a directory that + * does not exist holds a real, measured zero bytes. Every other errno stays + * unknown-with-reason. This mirrors usage-index's rootHealth vocabulary + * (ok / absent / degraded) so the two agree on what "we saw nothing" means. + * + * Presence describes only whether the ROOT could be read. A walk that hit an + * entry cap is still 'present' — the cap is a deliberate bound, not a read + * failure, and its effect is already carried by the Measurement's `partial`. */ +export function presenceOf(walk) { + if (!walk) return 'degraded'; + if (walk.status !== UNKNOWN) return 'present'; + return walk.reason === 'ENOENT' ? 'absent' : 'degraded'; +} + +/** Node figures for a root that may legitimately not exist. Absent → measured + * zero with presence 'absent'; unreadable → unknown with the errno. */ +export function rootMeasurements(walk, { asOf = null } = {}) { + const presence = presenceOf(walk); + if (presence === 'absent') { + return { presence, bytes: measured(0, { asOf }), files: measured(0, { asOf }) }; + } + return { presence, ...walkMeasurements(walk, { asOf }) }; +} diff --git a/src/lib/live/process-sessions.mjs b/src/lib/live/process-sessions.mjs index 5e1f338..c6c1c76 100644 --- a/src/lib/live/process-sessions.mjs +++ b/src/lib/live/process-sessions.mjs @@ -2,11 +2,22 @@ import { execFile } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { inspectGitWorkspace } from './git-workspace.mjs'; const execFileAsync = promisify(execFile); +// The Windows stand-in for `ps` + `lsof`. A text script invoked with -File, the +// same way the POSIX path invokes a binary with argv — no compiled artifact and +// no dependency. It lives BESIDE this module, not under the repo's scripts/, +// because it is a runtime asset rather than a dev tool: package.json's `files` +// ships `src/` wholesale, so colocation is what guarantees an npm-installed +// Windows user actually receives it. A checkout and an installed package +// therefore resolve the same sibling path. +const WIN32_SURVEY_SCRIPT = fileURLToPath( + new URL('./win-process-survey.ps1', import.meta.url)); + // Off by default (ADR-0023 §5/§4 precedent: AK_STATUSLINE_DEBUG). Set // AK_RUNTIME_DEBUG=1 to trace why a controller process was or wasn't // surfaced — which rows the ps survey saw, which were classified as a @@ -43,7 +54,12 @@ const HOST_NAMES = new Map([ const tokens = (command) => String(command ?? '').trim().split(/\s+/) .map((token) => token.replace(/^['"]|['"]$/g, '')); -const executableName = (value) => path.basename(String(value ?? '')).toLowerCase() +// win32.basename deliberately, on every platform: it splits on BOTH separators, +// so `C:\opt\bin\codex` and `/usr/local/bin/codex` both reduce to `codex`. The +// POSIX result is unchanged (a POSIX path has no backslash to split on), and +// the classification of a Windows command line stops depending on which OS +// happens to be running the classifier — which is what makes it unit-testable. +const executableName = (value) => path.win32.basename(String(value ?? '')).toLowerCase() .replace(/\.exe$/, ''); /** Identify only a controller executable or its supported Node launcher. */ @@ -100,6 +116,91 @@ function parseArgsByPid(output) { return commands; } +const isHostCandidate = (row) => { + const name = executableName(row.executable); + return HOST_NAMES.has(name) || name === 'node' || name === 'nodejs'; +}; + +/** + * Parse `win-process-survey.ps1 -Mode census`: the guaranteed Windows floor of + * pid, ppid, ISO-8601 UTC start, image name, CPU time and working set. Command + * is always '' — the census projection never asks for one. + * + * `startedAt` is an ISO instant here where the POSIX parsers keep `ps`'s raw + * `lstart` string. Both are opaque identity for the runtime key and both parse + * as a date; they are not interchangeable across platforms, and nothing + * compares them across one. + */ +export function parseWin32Census(output) { + const rows = []; + for (const line of String(output ?? '').split('\n')) { + const fields = line.replace(/\r$/, '').split('\t'); + if (fields.length < 6) continue; + const pid = Number(fields[0]); + const ppid = Number(fields[1]); + if (!Number.isInteger(pid) || !Number.isInteger(ppid)) continue; + const cpuTicks = Number(fields[4]); + const rssBytes = Number(fields[5]); + rows.push({ + pid, ppid, startedAt: fields[2], executable: fields[3], command: '', + // Win32_Process reports CPU in 100ns units; normalize once, here. + cpuMs: Number.isFinite(cpuTicks) ? cpuTicks / 10_000 : null, + rssBytes: Number.isFinite(rssBytes) ? rssBytes : null, + }); + } + return rows; +} + +/** + * Parse `-Mode commands`: `pid \t own|other|err \t value`. Three-valued on + * purpose — "someone else's process" and "we could not establish an owner" are + * different facts, and only the first is safe to treat as deliberately excluded. + */ +export function parseWin32Commands(output) { + const owned = new Map(); + const foreign = new Set(); + const failed = new Map(); + for (const line of String(output ?? '').split('\n')) { + const fields = line.replace(/\r$/, '').split('\t'); + if (fields.length < 3) continue; + const pid = Number(fields[0]); + if (!Number.isInteger(pid)) continue; + if (fields[1] === 'own') owned.set(pid, fields.slice(2).join(' ').trim()); + else if (fields[1] === 'other') foreign.add(pid); + else failed.set(pid, fields[2] || 'owner-probe-failed'); + } + return { owned, foreign, failed }; +} + +/** Parse `-Mode cwd`: `pid \t ok|err \t path-or-reason`. Never throws. */ +export function parseWin32Cwds(output) { + const found = new Map(); + const failures = new Map(); + for (const line of String(output ?? '').split('\n')) { + const fields = line.replace(/\r$/, '').split('\t'); + if (fields.length < 3) continue; + const pid = Number(fields[0]); + if (!Number.isInteger(pid)) continue; + if (fields[1] === 'ok' && fields[2]) found.set(pid, fields[2]); + else failures.set(pid, fields[2] || 'cwd-probe-failed'); + } + return { found, failures }; +} + +/** Parse `ps -o pid=,pcpu=,rss=`. RSS is kibibytes on both Linux and macOS. */ +export function parseProcessMetrics(output) { + const metrics = new Map(); + for (const line of String(output ?? '').split('\n')) { + const match = /^\s*(\d+)\s+(\d+(?:\.\d+)?)\s+(\d+)\s*$/.exec(line); + if (!match) continue; + metrics.set(Number(match[1]), { + cpuPercent: Number(match[2]), + rssBytes: Number(match[3]) * 1024, + }); + } + return metrics; +} + function rootControllers(rows) { const byPid = new Map(rows.map((row) => [row.pid, row])); const candidates = new Map(); @@ -163,6 +264,157 @@ async function darwinCwds(pids, run) { } } +function surveyFailure() { + return Object.assign(new Error('runtime process survey failed'), { + code: 'ERR_RUNTIME_PROCESS_SURVEY', + }); +} + +/** The current-user, argv-minimized POSIX survey, unchanged: a header pass that + * never carries argv, then argv for the pids that could actually be a host. */ +async function collectPosixRows({ execFileImpl, uid }) { + if (!Number.isInteger(uid) || uid < 0) { + throw Object.assign(new Error('runtime process survey cannot determine the current user'), { + code: 'ERR_RUNTIME_PROCESS_SURVEY', + }); + } + try { + const result = await execFileImpl('ps', [ + '-U', String(uid), '-x', '-o', 'pid=,ppid=,lstart=,comm=', + ], { encoding: 'utf8', timeout: 3000, maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, LC_ALL: 'C' } }); + const output = typeof result === 'string' ? result : result.stdout; + let rows = parseProcessHeaders(output); + runtimeDebug('survey', { uid, rowCount: rows.length }); + if (String(output ?? '').trim() && !rows.length) { + throw Object.assign(new Error('runtime process output was not understood'), { + code: 'ERR_RUNTIME_PROCESS_FORMAT', + }); + } + // argv can contain sensitive prompts/tokens. Fetch it only for executables + // that can actually be a supported controller or Node launcher. + const possible = rows.filter(isHostCandidate); + runtimeDebug('argv-candidates', { count: possible.length, pids: possible.map((r) => r.pid).join(',') }); + if (possible.length) { + const argsResult = await execFileImpl('ps', [ + '-p', possible.map((row) => row.pid).join(','), '-o', 'pid=,args=', + ], { encoding: 'utf8', timeout: 3000, maxBuffer: 1024 * 1024, + env: { ...process.env, LC_ALL: 'C' } }); + const commands = parseArgsByPid(typeof argsResult === 'string' ? argsResult : argsResult.stdout); + rows = rows.map((row) => commands.has(row.pid) + ? { ...row, command: commands.get(row.pid) } : row); + } + return rows; + } catch (error) { + runtimeDebug('survey-failed', { name: error?.name, code: error?.code }); + throw surveyFailure(); + } +} + +/** Windows PowerShell's absolute path, so PATH shadowing cannot redirect the + * survey. `powershell.exe` is in-box on every supported Windows; `pwsh` is + * deliberately not attempted because it is optional and may be absent. */ +function powershellCommand(env) { + const root = env?.SystemRoot || env?.WINDIR; + return root + ? path.join(root, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe') + : 'powershell.exe'; +} + +function powershellArgs(scriptPath, mode, pids = []) { + const args = [ + '-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', + '-File', scriptPath, '-Mode', mode, + ]; + // Numeric coercion is the injection guard (daemons.mjs precedent): nothing + // user-shaped can reach the script's WQL filter or its P/Invoke calls. + if (pids.length) args.push('-ProcessIds', pids.map((pid) => Number(pid)).join(',')); + return args; +} + +/** + * The Windows equivalent of collectPosixRows. Same two-phase privacy shape: a + * census that never reads a command line, then command lines for host-shaped + * pids the current user provably owns. + * + * Ownership has no cheap machine-wide query on Windows, so the census covers + * every process — pid/ppid are what the de-nesting walk needs, and a bare + * image name is not the sensitive field. The sensitive field, the command + * line, is read only after GetOwner proves the process is ours. A candidate we + * cannot attribute has its image name cleared, so classification cannot + * promote it to a controller. + */ +async function collectWin32Rows({ execFileImpl, scriptPath, env }) { + try { + const result = await execFileImpl(powershellCommand(env), powershellArgs(scriptPath, 'census'), { + encoding: 'utf8', timeout: 10_000, maxBuffer: 8 * 1024 * 1024, windowsHide: true }); + const output = typeof result === 'string' ? result : result.stdout; + const rows = parseWin32Census(output); + runtimeDebug('survey', { platform: 'win32', rowCount: rows.length }); + // Windows always has processes, so an empty census is a broken survey, not + // an idle machine — the same reasoning as the POSIX unparseable-output + // guard, except here even the empty-output case is unbelievable. + if (!rows.length) { + throw Object.assign(new Error('runtime process output was not understood'), { + code: 'ERR_RUNTIME_PROCESS_FORMAT', + }); + } + const possible = rows.filter(isHostCandidate); + runtimeDebug('argv-candidates', { count: possible.length, pids: possible.map((r) => r.pid).join(',') }); + if (!possible.length) return rows; + const owners = await execFileImpl( + powershellCommand(env), + powershellArgs(scriptPath, 'commands', possible.map((row) => row.pid)), + { encoding: 'utf8', timeout: 10_000, maxBuffer: 1024 * 1024, windowsHide: true }); + const { owned, failed } = parseWin32Commands( + typeof owners === 'string' ? owners : owners.stdout); + runtimeDebug('owner-probe', { owned: owned.size, unattributable: failed.size }); + return rows.map((row) => { + if (!isHostCandidate(row)) return row; + if (owned.has(row.pid)) return { ...row, command: owned.get(row.pid) }; + return { ...row, executable: '' }; + }); + } catch (error) { + runtimeDebug('survey-failed', { name: error?.name, code: error?.code }); + throw surveyFailure(); + } +} + +/** + * Best-effort per-process working directory on Windows. Never throws: a cwd we + * could not read costs the caller a project attribution, and must never cost it + * the census. Partial results are kept — one denied pid does not discard the + * rest. + */ +async function win32Cwds(pids, run, scriptPath, env) { + if (!pids.length) return { found: new Map(), failures: new Map() }; + try { + const result = await run( + powershellCommand(env), powershellArgs(scriptPath, 'cwd', pids), + { encoding: 'utf8', timeout: 20_000, maxBuffer: 1024 * 1024, windowsHide: true }); + return parseWin32Cwds(typeof result === 'string' ? result : result.stdout); + } catch (error) { + const partial = parseWin32Cwds(error?.stdout); + if (partial.found.size) return partial; + return { + found: new Map(), + failures: new Map(pids.map((pid) => [pid, 'cwd-survey-failed'])), + }; + } +} + +async function resolveCwds({ platform, pids, execFileImpl, scriptPath, env }) { + if (platform === 'linux') return { found: await linuxCwds(pids), failures: new Map() }; + if (platform === 'win32') return win32Cwds(pids, execFileImpl, scriptPath, env); + return { found: await darwinCwds(pids, execFileImpl), failures: new Map() }; +} + +async function collectRows({ platform, execFileImpl, uid, scriptPath, env }) { + return platform === 'win32' + ? collectWin32Rows({ execFileImpl, scriptPath, env }) + : collectPosixRows({ execFileImpl, uid }); +} + /** * Observe top-level Claude Code, Codex and OpenCode controller processes. * Child host CLIs remain part of their parent controller's execution graph. @@ -175,6 +427,8 @@ async function darwinCwds(pids, run) { * cwdByPid?: Map * inspectWorkspace?: typeof inspectGitWorkspace * uid?: number + * scriptPath?: string + * env?: NodeJS.ProcessEnv * }} [options] */ export async function listActiveHostSessions({ @@ -184,68 +438,39 @@ export async function listActiveHostSessions({ cwdByPid, inspectWorkspace = (cwd) => inspectGitWorkspace(cwd, { execFileImpl }), uid = process.getuid?.(), + scriptPath = WIN32_SURVEY_SCRIPT, + env = process.env, } = {}) { - if (platform === 'win32') { - throw Object.assign(new Error('runtime process survey is unsupported on Windows'), { - code: 'ERR_RUNTIME_UNSUPPORTED', - }); - } - let rows = processRows; - if (!rows) { - if (!Number.isInteger(uid) || uid < 0) { - throw Object.assign(new Error('runtime process survey cannot determine the current user'), { - code: 'ERR_RUNTIME_PROCESS_SURVEY', - }); - } - try { - const result = await execFileImpl('ps', [ - '-U', String(uid), '-x', '-o', 'pid=,ppid=,lstart=,comm=', - ], { encoding: 'utf8', timeout: 3000, maxBuffer: 4 * 1024 * 1024, - env: { ...process.env, LC_ALL: 'C' } }); - const output = typeof result === 'string' ? result : result.stdout; - rows = parseProcessHeaders(output); - runtimeDebug('survey', { uid, rowCount: rows.length }); - if (String(output ?? '').trim() && !rows.length) { - throw Object.assign(new Error('runtime process output was not understood'), { - code: 'ERR_RUNTIME_PROCESS_FORMAT', - }); - } - // argv can contain sensitive prompts/tokens. Fetch it only for executables - // that can actually be a supported controller or Node launcher. - const possible = rows.filter((row) => { - const name = executableName(row.executable); - return HOST_NAMES.has(name) || name === 'node' || name === 'nodejs'; - }); - runtimeDebug('argv-candidates', { count: possible.length, pids: possible.map((r) => r.pid).join(',') }); - if (possible.length) { - const argsResult = await execFileImpl('ps', [ - '-p', possible.map((row) => row.pid).join(','), '-o', 'pid=,args=', - ], { encoding: 'utf8', timeout: 3000, maxBuffer: 1024 * 1024, - env: { ...process.env, LC_ALL: 'C' } }); - const commands = parseArgsByPid(typeof argsResult === 'string' ? argsResult : argsResult.stdout); - rows = rows.map((row) => commands.has(row.pid) - ? { ...row, command: commands.get(row.pid) } : row); - } - } catch (error) { - runtimeDebug('survey-failed', { name: error?.name, code: error?.code }); - throw Object.assign(new Error('runtime process survey failed'), { - code: 'ERR_RUNTIME_PROCESS_SURVEY', + const rows = processRows + ?? await collectRows({ platform, execFileImpl, uid, scriptPath, env }); + const controllers = rootControllers(rows); + const pids = controllers.map((row) => row.pid); + let cwds = cwdByPid; + if (!cwds) { + const resolved = await resolveCwds({ platform, pids, execFileImpl, scriptPath, env }); + cwds = resolved.found; + // A session without a workspace is not a session this consumer can use, so + // an across-the-board cwd failure degrades the source rather than reporting + // a healthy empty survey. The footprint census takes the opposite trade — + // see surveyHostProcesses. + if (pids.length && !cwds.size && resolved.failures.size) { + throw Object.assign(new Error('runtime cwd survey failed'), { + code: 'ERR_RUNTIME_CWD_SURVEY', }); } } - const controllers = rootControllers(rows); - const pids = controllers.map((row) => row.pid); - const cwds = cwdByPid ?? (platform === 'linux' - ? await linuxCwds(pids) : await darwinCwds(pids, execFileImpl)); for (const pid of pids) runtimeDebug('cwd', { pid, found: cwds.has(pid), cwd: cwds.get(pid) ?? '' }); const workspaceByCwd = new Map(); await Promise.all([...new Set(cwds.values())].map(async (cwd) => { try { workspaceByCwd.set(cwd, await inspectWorkspace(cwd)); } catch { workspaceByCwd.set(cwd, null); } })); + // Explicitly platform-flavored: `C:\repo` is absolute for a win32 survey even + // when this process is running elsewhere (which is how win32 is unit-tested). + const isAbsoluteFor = platform === 'win32' ? path.win32.isAbsolute : path.posix.isAbsolute; const sessions = controllers.flatMap((row) => { const cwd = cwds.get(row.pid); - if (typeof cwd !== 'string' || !path.isAbsolute(cwd)) { + if (typeof cwd !== 'string' || !isAbsoluteFor(cwd)) { runtimeDebug('drop-no-cwd', { pid: row.pid, host: row.host }); return []; } @@ -257,3 +482,128 @@ export async function listActiveHostSessions({ runtimeDebug('result', { sessionCount: sessions.length }); return sessions; } + +/** Count every descendant of a controller: tool children, MCP servers, shells. + * Cycle-guarded the same way the ancestor walk is. */ +function countDescendants(rows, rootPids) { + const children = new Map(); + for (const row of rows) { + if (!children.has(row.ppid)) children.set(row.ppid, []); + children.get(row.ppid).push(row.pid); + } + const seen = new Set(rootPids); + const queue = [...rootPids]; + let count = 0; + while (queue.length) { + for (const child of children.get(queue.shift()) ?? []) { + if (seen.has(child)) continue; + seen.add(child); + count += 1; + queue.push(child); + } + } + return count; +} + +/** Second, argv-free `ps` pass for the two figures the header survey omits. + * Never throws: a census without CPU/RSS is still a census. */ +async function posixMetrics(pids, run) { + if (!pids.length) return new Map(); + try { + const result = await run('ps', ['-p', pids.join(','), '-o', 'pid=,pcpu=,rss='], { + encoding: 'utf8', timeout: 3000, maxBuffer: 1024 * 1024, + env: { ...process.env, LC_ALL: 'C' } }); + return parseProcessMetrics(typeof result === 'string' ? result : result.stdout); + } catch (error) { + return parseProcessMetrics(error?.stdout); + } +} + +/** + * Point-in-time resource census of the same controller processes + * `listActiveHostSessions` finds — host, pid, uptime, CPU%, RSS, and the bound + * project when the platform can attribute one. + * + * The deliberate divergence from `listActiveHostSessions`: that function drops + * a controller whose cwd it cannot read, because a session with no workspace is + * not a usable session. A census must not, because a process we can measure but + * cannot attribute is still consuming this machine's memory, and dropping it + * would understate the totals the whole System area is denominated in. Such a + * row keeps every measured field and carries `cwd: null` with a `cwdReason`. + * + * CPU% is the same quantity on both platforms — CPU time over process lifetime, + * as a percentage of one core — so `ps`'s %CPU and Win32_Process's kernel+user + * ticks over uptime are directly comparable. + * + * @param {{ + * platform?: NodeJS.Platform, execFileImpl?: typeof execFileAsync, + * processRows?: Array, cwdByPid?: Map, + * metricsByPid?: Map, + * uid?: number, scriptPath?: string, env?: NodeJS.ProcessEnv, now?: number + * }} [options] + */ +export async function surveyHostProcesses({ + platform = process.platform, + execFileImpl = execFileAsync, + processRows, + cwdByPid, + metricsByPid, + uid = process.getuid?.(), + scriptPath = WIN32_SURVEY_SCRIPT, + env = process.env, + now = Date.now(), +} = {}) { + const rows = processRows + ?? await collectRows({ platform, execFileImpl, uid, scriptPath, env }); + const controllers = rootControllers(rows); + const pids = controllers.map((row) => row.pid); + // Whatever the platform's cwd probe does — lsof failing outright, the PEB + // read being blocked — the census survives it and reports the reason per row. + let resolved = { found: cwdByPid ?? new Map(), failures: new Map() }; + if (!cwdByPid) { + try { resolved = await resolveCwds({ platform, pids, execFileImpl, scriptPath, env }); } + catch (error) { + resolved = { found: new Map(), failures: new Map(pids.map((pid) => [pid, + error?.code === 'ERR_RUNTIME_CWD_SURVEY' ? 'cwd-survey-failed' : 'cwd-unavailable'])) }; + } + } + const metrics = metricsByPid + ?? (platform === 'win32' ? new Map() : await posixMetrics(pids, execFileImpl)); + const isAbsoluteFor = platform === 'win32' ? path.win32.isAbsolute : path.posix.isAbsolute; + + const processes = controllers.map((row) => { + const startedMs = Date.parse(row.startedAt); + const uptimeMs = Number.isFinite(startedMs) && startedMs <= now ? now - startedMs : null; + const measured = metrics.get(row.pid); + // Win32_Process reports CPU as consumed time; ps reports it as a lifetime + // rate. Convert the former into the latter so one column means one thing. + const cpuPercent = measured?.cpuPercent ?? ( + Number.isFinite(row.cpuMs) && uptimeMs > 0 ? (row.cpuMs / uptimeMs) * 100 : null); + const rssBytes = measured?.rssBytes ?? (Number.isFinite(row.rssBytes) ? row.rssBytes : null); + const cwd = resolved.found.get(row.pid); + const attributable = typeof cwd === 'string' && isAbsoluteFor(cwd); + return { + host: row.host, + pid: row.pid, + ppid: row.ppid, + startedAt: row.startedAt, + uptimeMs, + cpuPercent: Number.isFinite(cpuPercent) ? cpuPercent : null, + rssBytes: Number.isFinite(rssBytes) ? rssBytes : null, + cwd: attributable ? cwd : null, + cwdReason: attributable ? null : (resolved.failures.get(row.pid) ?? 'cwd-unavailable'), + }; + }).sort((left, right) => left.pid - right.pid); + + runtimeDebug('census', { + platform, controllers: processes.length, + attributed: processes.filter((entry) => entry.cwd).length, + }); + return { + platform, + observedAt: new Date(now).toISOString(), + processes, + childProcessCount: countDescendants(rows, new Set(pids)), + surveyedProcessCount: rows.length, + }; +} diff --git a/src/lib/live/win-process-survey.ps1 b/src/lib/live/win-process-survey.ps1 new file mode 100644 index 0000000..fa27dac --- /dev/null +++ b/src/lib/live/win-process-survey.ps1 @@ -0,0 +1,249 @@ +<# + agentic-kit — Windows host-process survey. + + The POSIX runtime survey shells out to `ps` and `lsof`; Windows has no + equivalent single binary, so this script is that binary's stand-in. It is + plain text, invoked with -File exactly the way `ps` is invoked with argv: + nothing here is compiled, installed, or persisted, and it takes no npm + dependency. + + Three modes, deliberately separated so the privacy-sensitive read and the + fragile read are each isolated from the guaranteed floor: + + -Mode census every process's pid, ppid, start time, image + name, CPU time and working set. No command + lines. This is the GUARANTEED floor: if the + two modes below fail entirely, the caller + still has a complete resource census. + -Mode commands -ProcessIds command line for the named pids, and ONLY + for pids the current user actually owns. + Ownership is proven with GetOwner, never + assumed from session id — a process we + cannot attribute is reported as such, never + silently included. + -Mode cwd -ProcessIds best-effort true working directory, read out + of each process's PEB. This is the only part + that can fail for environmental reasons (AV + blocking Add-Type, constrained language mode, + access denied, bitness mismatch) and every + one of those failures is reported per-pid as + an `err` row rather than raised — the caller + must be able to keep the census when this + mode returns nothing useful. + + Every mode emits tab-separated lines and exits 0. Tabs and newlines are + stripped out of command lines before emission so a row is always one line + with a fixed field count. +#> +[CmdletBinding()] +param( + [ValidateSet('census', 'commands', 'cwd')] + [string]$Mode = 'census', + + # Comma-separated decimal pids. The caller already numeric-coerces these; + # the regex below is the second guard, so nothing user-shaped can reach a + # WQL filter or a P/Invoke call. + [string]$ProcessIds = '' +) + +$ErrorActionPreference = 'Stop' +try { [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch { } + +function Get-RequestedIds { + param([string]$Raw) + $ids = @() + foreach ($part in ($Raw -split ',')) { + $trimmed = $part.Trim() + if ($trimmed -match '^\d+$') { $ids += [int]$trimmed } + } + return $ids +} + +if ($Mode -eq 'census') { + # CommandLine is deliberately absent from this projection: it is the one + # field that can hold a pasted prompt or a token, and the floor never + # needs it. + $props = @( + 'ProcessId', 'ParentProcessId', 'Name', 'CreationDate', + 'KernelModeTime', 'UserModeTime', 'WorkingSetSize' + ) + foreach ($proc in (Get-CimInstance -ClassName Win32_Process -Property $props)) { + $created = '' + if ($proc.CreationDate) { + $created = ([datetime]$proc.CreationDate).ToUniversalTime().ToString('yyyy-MM-dd\THH:mm:ss\Z') + } + $cpu = [uint64]0 + if ($proc.KernelModeTime) { $cpu = $cpu + [uint64]$proc.KernelModeTime } + if ($proc.UserModeTime) { $cpu = $cpu + [uint64]$proc.UserModeTime } + $rss = [uint64]0 + if ($proc.WorkingSetSize) { $rss = [uint64]$proc.WorkingSetSize } + $name = ([string]$proc.Name) -replace '\s+', '_' + "$($proc.ProcessId)`t$($proc.ParentProcessId)`t$created`t$name`t$cpu`t$rss" + } + exit 0 +} + +if ($Mode -eq 'commands') { + $ids = Get-RequestedIds -Raw $ProcessIds + if ($ids.Count -eq 0) { exit 0 } + $filter = (($ids | ForEach-Object { "ProcessId=$_" }) -join ' OR ') + $me = [string]$env:USERNAME + foreach ($proc in (Get-CimInstance -ClassName Win32_Process -Filter $filter)) { + $owner = $null + try { $owner = Invoke-CimMethod -InputObject $proc -MethodName GetOwner -ErrorAction Stop } + catch { $owner = $null } + if (-not $owner -or $owner.ReturnValue -ne 0) { + # Unattributable, not foreign — the caller must be able to tell the + # difference between "someone else's process" and "we could not ask". + "$($proc.ProcessId)`terr`towner-probe-failed" + continue + } + if ([string]$owner.User -ne $me) { + "$($proc.ProcessId)`tother`t" + continue + } + $commandLine = ([string]$proc.CommandLine) -replace '[\r\n\t]+', ' ' + "$($proc.ProcessId)`town`t$commandLine" + } + exit 0 +} + +# -Mode cwd. Windows exposes no supported API for another process's current +# directory, so this walks the documented layout the debugger tooling walks: +# NtQueryInformationProcess -> PEB -> RTL_USER_PROCESS_PARAMETERS -> +# CurrentDirectory.DosPath, reading each hop with ReadProcessMemory. The offsets +# below are the 64-bit-process layout only; a bitness mismatch between this +# reader and the target is DETECTED and reported, never read through with the +# wrong offsets, because a plausible-looking wrong path is worse than no path. +$ids = Get-RequestedIds -Raw $ProcessIds +if ($ids.Count -eq 0) { exit 0 } + +$source = @' +using System; +using System.Text; +using System.Runtime.InteropServices; + +public static class AkProcessCwd +{ + [StructLayout(LayoutKind.Sequential)] + private struct PROCESS_BASIC_INFORMATION + { + public IntPtr ExitStatus; + public IntPtr PebBaseAddress; + public IntPtr AffinityMask; + public IntPtr BasePriority; + public IntPtr UniqueProcessId; + public IntPtr InheritedFromUniqueProcessId; + } + + [DllImport("ntdll.dll")] + private static extern int NtQueryInformationProcess(IntPtr handle, int infoClass, + ref PROCESS_BASIC_INFORMATION info, int length, out int written); + + [DllImport("ntdll.dll")] + private static extern int NtQueryInformationProcess(IntPtr handle, int infoClass, + out IntPtr info, int length, out int written); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr OpenProcess(int access, bool inherit, int pid); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool CloseHandle(IntPtr handle); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool ReadProcessMemory(IntPtr handle, IntPtr address, + byte[] buffer, IntPtr size, out IntPtr read); + + private const int ProcessBasicInformation = 0; + private const int ProcessWow64Information = 26; + + // PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_VM_READ. Deliberately not + // PROCESS_ALL_ACCESS: this probe reads six pointers and one string. + private const int Access = 0x1000 | 0x0010; + + private const long PebProcessParameters = 0x20; + private const long ParametersCurrentDirectory = 0x38; + + private static IntPtr Offset(IntPtr basePtr, long delta) + { + return new IntPtr(basePtr.ToInt64() + delta); + } + + private static byte[] ReadBytes(IntPtr handle, IntPtr address, int size) + { + byte[] buffer = new byte[size]; + IntPtr read; + if (!ReadProcessMemory(handle, address, buffer, new IntPtr(size), out read)) { return null; } + if (read.ToInt64() != size) { return null; } + return buffer; + } + + public static string Read(int pid) + { + if (IntPtr.Size != 8) { return "err\treader-not-64bit"; } + IntPtr handle = OpenProcess(Access, false, pid); + if (handle == IntPtr.Zero) { return "err\topen-denied"; } + try + { + IntPtr wow64 = IntPtr.Zero; + int written; + int wowStatus = NtQueryInformationProcess(handle, ProcessWow64Information, + out wow64, IntPtr.Size, out written); + if (wowStatus == 0 && wow64 != IntPtr.Zero) { return "err\twow64-mismatch"; } + + PROCESS_BASIC_INFORMATION info = new PROCESS_BASIC_INFORMATION(); + int status = NtQueryInformationProcess(handle, ProcessBasicInformation, + ref info, Marshal.SizeOf(typeof(PROCESS_BASIC_INFORMATION)), out written); + if (status != 0) { return "err\tquery-failed"; } + if (info.PebBaseAddress == IntPtr.Zero) { return "err\tno-peb"; } + + byte[] pointer = ReadBytes(handle, Offset(info.PebBaseAddress, PebProcessParameters), 8); + if (pointer == null) { return "err\tpeb-read-failed"; } + long parameters = BitConverter.ToInt64(pointer, 0); + if (parameters == 0) { return "err\tno-parameters"; } + + // CurrentDirectory.DosPath is a UNICODE_STRING: Length, MaximumLength, + // 4 bytes of padding on x64, then the wide-char buffer pointer. + byte[] header = ReadBytes(handle, + new IntPtr(parameters + ParametersCurrentDirectory), 16); + if (header == null) { return "err\tparameters-read-failed"; } + int length = BitConverter.ToUInt16(header, 0); + long buffer = BitConverter.ToInt64(header, 8); + if (length <= 0 || length > 8192 || buffer == 0) { return "err\tempty-cwd"; } + + byte[] raw = ReadBytes(handle, new IntPtr(buffer), length); + if (raw == null) { return "err\tcwd-read-failed"; } + string value = Encoding.Unicode.GetString(raw).TrimEnd('\0').Trim(); + if (value.Length == 0) { return "err\tempty-cwd"; } + value = value.TrimEnd('\\'); + if (value.Length == 2 && value[1] == ':') { value = value + "\\"; } + if (value.IndexOf('\t') >= 0 || value.IndexOf('\n') >= 0) { return "err\tunexpected-cwd"; } + return "ok\t" + value; + } + catch { return "err\tprobe-failed"; } + finally { CloseHandle(handle); } + } +} +'@ + +$ready = $false +try { + Add-Type -TypeDefinition $source -Language CSharp -ErrorAction Stop + $ready = $true +} catch { + $ready = $false +} + +if (-not $ready) { + # AV interception, constrained language mode, or no in-box compiler. Say so + # for every pid; the caller keeps its census and drops only the project + # attribution. + foreach ($id in $ids) { "$id`terr`tcompile-failed" } + exit 0 +} + +foreach ($id in $ids) { + try { "$id`t$([AkProcessCwd]::Read($id))" } + catch { "$id`terr`tprobe-failed" } +} +exit 0 diff --git a/tests/kit/footprint-collectors.test.mjs b/tests/kit/footprint-collectors.test.mjs new file mode 100644 index 0000000..e3598ee --- /dev/null +++ b/tests/kit/footprint-collectors.test.mjs @@ -0,0 +1,989 @@ +// System-area (machine-footprint) collectors: the bounded walker and the +// Measurement vocabulary every collector reports in, then Storage, Projects and +// Catalog over real temp-dir fixtures. +// +// Two rules these tests hold themselves to, because the collectors default to +// the real ~/.claude, ~/.codex and npm caches: every fixture path is created +// under mkdtempSync, and every collector is handed an fs restricted to that +// fixture. A test that reached the developer's home directory fails with ENOENT +// rather than passing slowly against whatever happens to be installed. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + walkTree, walkMeasurements, rootMeasurements, presenceOf, statNode, + measured, unknown, carriedForward, sumMeasurements, isMeasured, hasValue, + MEASURED, UNKNOWN, WALK_LIMITS, +} from '../../src/lib/footprint/walk.mjs'; +import * as storageModule from '../../src/lib/footprint/storage.mjs'; +import { + collectStorage, worktreeReclaimables, buildGrowth, localDay, STORAGE_DEFAULTS, +} from '../../src/lib/footprint/storage.mjs'; +import { + collectProjects, countLines, describeRemote, measureProject, nodeModulesRoots, + parseGitRemote, projectRemote, LOC_EXCLUSIONS, +} from '../../src/lib/footprint/projects.mjs'; +import { collectCatalog, tomlTableNames } from '../../src/lib/footprint/catalog.mjs'; + +const DAY = 86_400_000; + +/** A fixture root that is removed when the test ends, whatever the outcome. */ +function fixture(t, name) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `ak-footprint-${name}-`)); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +/** Write one fixture file and return its byte length, so a test's expected + * totals are derived from what it actually created rather than restated. */ +function write(file, content) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content); + fs.writeFileSync(file, buffer); + return buffer.length; +} + +const touch = (file, ms) => fs.utimesSync(file, new Date(ms), new Date(ms)); + +/** + * Real fs, fenced to `root`, with an optional set of directories made + * unreadable. The fence is what proves a collector never reached the real home; + * the deny list is the portable stand-in for `chmod 000`, which Windows cannot + * reproduce and a root-run CI would ignore. + */ +function fixtureFs(root, { deny = [], denyCode = 'EACCES' } = {}) { + const allowed = path.resolve(root); + const denied = deny.map((dir) => path.resolve(dir)); + const guard = (target) => { + const resolved = path.resolve(String(target)); + if (resolved === allowed || resolved.startsWith(allowed + path.sep)) return resolved; + throw Object.assign(new Error(`ENOENT: outside the fixture (${resolved})`), { code: 'ENOENT' }); + }; + return { + ...fs, + readdirSync: (target, options) => { + const resolved = guard(target); + if (denied.includes(resolved)) { + throw Object.assign(new Error(`${denyCode}: ${resolved}`), { code: denyCode }); + } + return fs.readdirSync(resolved, options); + }, + lstatSync: (target) => fs.lstatSync(guard(target)), + statSync: (target) => fs.statSync(guard(target)), + readFileSync: (target, options) => fs.readFileSync(guard(target), options), + openSync: (target, flags) => fs.openSync(guard(target), flags), + }; +} + +// ── the bounded walker ──────────────────────────────────────────────────────── + +test('the walker returns bytes, file count and newest mtime for a real tree', (t) => { + const root = fixture(t, 'walk'); + const a = write(path.join(root, 'a.txt'), 'a'.repeat(100)); + const b = write(path.join(root, 'nested', 'b.txt'), 'b'.repeat(30)); + const newest = Date.now() - 5 * DAY; + touch(path.join(root, 'a.txt'), newest - DAY); + touch(path.join(root, 'nested', 'b.txt'), newest); + + const result = walkTree(root); + assert.equal(result.status, MEASURED); + assert.equal(result.bytes, a + b); + assert.equal(result.files, 2); + assert.equal(Math.round(result.newestMtimeMs), newest); + assert.equal(result.complete, true); + assert.equal(result.truncated, false); + assert.equal(result.degradedCount, 0); +}); + +test('the walker never follows a symlink, so a symlink cycle terminates', (t) => { + const root = fixture(t, 'walk-symlink'); + const real = write(path.join(root, 'tree', 'real.txt'), 'x'.repeat(64)); + try { + fs.symlinkSync(root, path.join(root, 'tree', 'loop'), 'dir'); + fs.symlinkSync(path.join(root, 'tree'), path.join(root, 'sideways'), 'dir'); + fs.symlinkSync(path.join(root, 'tree', 'real.txt'), path.join(root, 'alias.txt'), 'file'); + } catch { + t.skip('this platform does not permit unprivileged symlink creation'); + return; + } + + const result = walkTree(root); + assert.equal(result.status, MEASURED); + // The cycle would never terminate if `loop` were followed, and the aliases + // would double- and triple-count `real.txt`'s bytes if they were measured. + assert.equal(result.bytes, real); + assert.equal(result.files, 1); + assert.equal(result.symlinksSkipped, 3); + assert.equal(result.complete, true); + + // A symlinked ROOT is refused outright rather than resolved to its target. + const head = walkTree(path.join(root, 'sideways')); + assert.equal(head.status, UNKNOWN); + assert.match(head.reason, /symlink/); + assert.equal(head.bytes, null); +}); + +test('the walker respects the depth cap and reports the truncation', (t) => { + const root = fixture(t, 'walk-depth'); + const shallow = write(path.join(root, 'top.txt'), 'a'.repeat(10)); + write(path.join(root, 'one', 'two', 'three', 'deep.txt'), 'b'.repeat(1000)); + + const result = walkTree(root, { maxDepth: 1 }); + assert.equal(result.truncated, true); + assert.equal(result.truncatedBy, 'depth'); + assert.equal(result.complete, false); + assert.equal(result.bytes, shallow, 'a capped walk reports a floor, not the total'); + // A floor is still a MEASUREMENT — it is flagged partial, never downgraded to + // unknown and never rounded to zero. + const { bytes } = walkMeasurements(result); + assert.equal(bytes.status, MEASURED); + assert.equal(bytes.partial, true); +}); + +test('the walker respects the entry cap', (t) => { + const root = fixture(t, 'walk-entries'); + for (let i = 0; i < 10; i++) write(path.join(root, `f${i}.txt`), 'x'.repeat(8)); + + const result = walkTree(root, { maxEntries: 3 }); + assert.equal(result.truncated, true); + assert.equal(result.truncatedBy, 'entries'); + assert.equal(result.complete, false); + assert.ok(result.entriesSeen <= 3, `entriesSeen ${result.entriesSeen} exceeded the cap`); + assert.ok(result.files < 10); + assert.equal(WALK_LIMITS.maxEntries > 3, true, 'the shipped cap is not the test cap'); +}); + +test('one unreadable subtree degrades that node while its siblings still report', (t) => { + const root = fixture(t, 'walk-degrade'); + const ok = write(path.join(root, 'readable', 'a.txt'), 'a'.repeat(40)); + const alsoOk = write(path.join(root, 'other', 'b.txt'), 'b'.repeat(20)); + write(path.join(root, 'denied', 'secret.txt'), 'c'.repeat(9999)); + + const denied = path.join(root, 'denied'); + const result = walkTree(root, { fsImpl: fixtureFs(root, { deny: [denied] }) }); + assert.equal(result.status, MEASURED, 'a degraded child never unknowns the whole walk'); + assert.equal(result.bytes, ok + alsoOk, 'siblings keep their measured bytes'); + assert.equal(result.files, 2); + assert.equal(result.complete, false); + assert.deepEqual(result.degraded, [{ path: denied, reason: 'EACCES' }]); +}); + +test('the degraded COUNT is exact while the retained sample stays capped', (t) => { + const root = fixture(t, 'walk-degrade-cap'); + const denied = []; + for (let i = 0; i < 5; i++) { + write(path.join(root, `d${i}`, 'x.txt'), 'x'); + denied.push(path.join(root, `d${i}`)); + } + const result = walkTree(root, { maxDegraded: 2, fsImpl: fixtureFs(root, { deny: denied }) }); + assert.equal(result.degradedCount, 5); + assert.equal(result.degraded.length, 2); +}); + +test('an unreadable root is unknown-with-reason, never a zero-byte directory', (t) => { + const root = fixture(t, 'walk-root-denied'); + write(path.join(root, 'a.txt'), 'a'.repeat(100)); + + const result = walkTree(root, { fsImpl: fixtureFs(root, { deny: [root] }) }); + assert.equal(result.status, UNKNOWN); + assert.equal(result.reason, 'EACCES'); + assert.equal(result.bytes, null, 'an EACCES directory is not an empty one'); + assert.equal(result.files, null); + assert.equal(result.complete, false); + assert.equal(presenceOf(result), 'degraded'); + + const { bytes, files } = rootMeasurements(result); + for (const figure of [bytes, files]) { + assert.equal(figure.status, UNKNOWN); + assert.equal(figure.value, null); + assert.equal(figure.reason, 'EACCES'); + assert.notEqual(figure.value, 0); + } +}); + +test('a file root is a node in its own right, and a vanished file degrades only itself', (t) => { + const root = fixture(t, 'walk-file-root'); + const history = path.join(root, 'history.jsonl'); + const size = write(history, 'a'.repeat(72)); + + // Several known roots are single files (history.jsonl, the opencode store). + const single = walkTree(history); + assert.equal(single.status, MEASURED); + assert.deepEqual({ bytes: single.bytes, files: single.files }, { bytes: size, files: 1 }); + assert.equal(single.complete, true); + const filtered = walkTree(history, { acceptFile: () => false }); + assert.equal(filtered.files, 0, 'a deliberate filter is a scope, not a failure'); + assert.equal(filtered.complete, true); + + // A file that disappears between readdir and lstat is one degraded node. + const kept = write(path.join(root, 'kept.log'), 'b'.repeat(10)); + const racing = path.join(root, 'gone.log'); + write(racing, 'c'.repeat(999)); + const base = fixtureFs(root); + const result = walkTree(root, { + fsImpl: { + ...base, + lstatSync: (target) => { + if (path.resolve(String(target)) === racing) { + throw Object.assign(new Error('gone'), { code: 'ENOENT' }); + } + return base.lstatSync(target); + }, + }, + }); + assert.equal(result.bytes, size + kept); + assert.equal(result.files, 2); + assert.deepEqual(result.degraded, [{ path: racing, reason: 'ENOENT' }]); + assert.equal(result.complete, false); +}); + +test('a walker callback exception surfaces instead of reading as an unreadable subtree', (t) => { + const root = fixture(t, 'walk-onfile-throws'); + write(path.join(root, 'a.txt'), 'a'); + assert.throws(() => walkTree(root, { + onFile: () => { throw new Error('collector bug'); }, + }), /collector bug/); +}); + +// ── measurement provenance ──────────────────────────────────────────────────── + +test('an unmeasured value reads unknown-with-reason and never 0', () => { + const figure = unknown('EACCES'); + assert.equal(figure.status, UNKNOWN); + assert.equal(figure.value, null); + assert.equal(figure.reason, 'EACCES'); + assert.equal(hasValue(figure), false); + assert.equal(isMeasured(figure), false); + // Even an unknown constructed without a stated cause carries one, because a + // reasonless unknown is indistinguishable from a forgotten zero. + assert.equal(unknown('').reason, 'unmeasured'); + assert.equal(unknown(null).reason, 'unmeasured'); +}); + +test('a real empty directory reads as a MEASURED zero, and a missing one as absent', (t) => { + const root = fixture(t, 'measure-empty'); + const empty = path.join(root, 'empty'); + fs.mkdirSync(empty); + + const walked = walkTree(empty); + assert.equal(walked.status, MEASURED); + assert.equal(walked.bytes, 0); + assert.equal(presenceOf(walked), 'present'); + const present = rootMeasurements(walked); + assert.equal(present.presence, 'present'); + assert.deepEqual( + { value: present.bytes.value, status: present.bytes.status, reason: present.bytes.reason }, + { value: 0, status: MEASURED, reason: null }, + 'an empty directory really does hold zero bytes', + ); + + // Absence is a measured zero too — a root that does not exist holds nothing. + const gone = rootMeasurements(walkTree(path.join(root, 'nope'))); + assert.equal(gone.presence, 'absent'); + assert.equal(gone.bytes.status, MEASURED); + assert.equal(gone.bytes.value, 0); + + // The distinction the whole domain turns on: same 0 on screen, different + // provenance underneath, and an unreadable root gets neither. + const denied = rootMeasurements(walkTree(empty, { fsImpl: fixtureFs(root, { deny: [empty] }) })); + assert.equal(denied.presence, 'degraded'); + assert.equal(denied.bytes.status, UNKNOWN); + assert.equal(denied.bytes.value, null); +}); + +test('sums stay honest: empty is zero, all-unknown is unknown, mixed is partial', () => { + assert.equal(sumMeasurements([]).value, 0); + assert.equal(sumMeasurements([]).status, MEASURED); + + const allUnknown = sumMeasurements([unknown('EACCES'), unknown('EPERM')]); + assert.equal(allUnknown.status, UNKNOWN); + assert.equal(allUnknown.value, null); + + const mixed = sumMeasurements([measured(10), unknown('EACCES'), measured(5)]); + assert.equal(mixed.status, MEASURED); + assert.equal(mixed.value, 15, 'an unmeasured input never contributes a 0'); + assert.equal(mixed.partial, true, 'a sum missing an input is a floor'); + + const clean = sumMeasurements([measured(10), measured(5)]); + assert.equal(clean.partial, false); + assert.equal(sumMeasurements([measured(1, { partial: true }), measured(2)]).partial, true); +}); + +test('a carried-forward figure keeps the scan that produced it, not the current time', () => { + const then = Date.now() - 3 * DAY; + const figure = carriedForward(4096, then); + assert.equal(figure.status, 'carried-forward'); + assert.equal(figure.asOf, then); + assert.equal(isMeasured(figure), false, 'carried forward is not "this scan measured it"'); + assert.equal(hasValue(figure), true); +}); + +test('statNode reports a symlink as a symlink rather than measuring its target', (t) => { + const root = fixture(t, 'statnode'); + const file = path.join(root, 'real.txt'); + write(file, 'a'.repeat(12)); + const node = statNode(file); + assert.equal(node.kind, 'file'); + assert.equal(node.bytes, 12); + + try { fs.symlinkSync(file, path.join(root, 'alias.txt'), 'file'); } catch { + t.skip('this platform does not permit unprivileged symlink creation'); + return; + } + const alias = statNode(path.join(root, 'alias.txt')); + assert.equal(alias.kind, 'symlink'); + assert.equal(alias.bytes, null); + assert.equal(statNode(path.join(root, 'missing')).reason, 'ENOENT'); +}); + +// ── storage ─────────────────────────────────────────────────────────────────── + +/** Category → host → project → session fixture with known sizes and mtimes. */ +function storageFixture(t) { + const root = fixture(t, 'storage'); + const now = Date.now(); + const claudeProjects = path.join(root, 'claude', 'projects'); + const codexSessions = path.join(root, 'codex', 'sessions'); + const akConfig = path.join(root, 'ak', 'config'); + + const sizes = { + recent: write(path.join(claudeProjects, '-repos-keel', 'recent.jsonl'), 'a'.repeat(100)), + aged: write(path.join(claudeProjects, '-repos-keel', 'aged.jsonl'), 'b'.repeat(50)), + rollout: write(path.join(codexSessions, '2026', '08', '06', 'rollout.jsonl'), 'c'.repeat(30)), + index: write(path.join(akConfig, 'usage-index.json'), 'd'.repeat(10)), + }; + touch(path.join(claudeProjects, '-repos-keel', 'recent.jsonl'), now - DAY); + touch(path.join(claudeProjects, '-repos-keel', 'aged.jsonl'), now - 200 * DAY); + touch(path.join(codexSessions, '2026', '08', '06', 'rollout.jsonl'), now - 2 * DAY); + touch(path.join(akConfig, 'usage-index.json'), now - 400 * DAY); + + const roots = [ + { id: 'claude-transcripts', category: 'transcripts', host: 'claude', + label: 'session transcripts', path: claudeProjects, layout: 'claude-projects' }, + { id: 'codex-transcripts', category: 'transcripts', host: 'codex', + label: 'session rollouts', path: codexSessions, layout: 'flat-sessions' }, + { id: 'ak-config', category: 'kit-caches', host: 'agentic-kit', + label: 'config, indexes & snapshots', path: akConfig, layout: 'tree' }, + ]; + return { root, now, roots, sizes, claudeProjects, codexSessions }; +} + +const nodeAt = (nodes, key) => nodes.find((node) => node.key === key); + +test('storage sums the category → host → project → session tree', (t) => { + const { root, now, roots, sizes, claudeProjects } = storageFixture(t); + const result = collectStorage({ + roots, projects: [], now: () => now, detectWorktrees: false, fsImpl: fixtureFs(root), + }); + + const transcripts = nodeAt(result.categories, 'transcripts'); + assert.equal(transcripts.bytes.value, sizes.recent + sizes.aged + sizes.rollout); + assert.equal(transcripts.files.value, 3); + + const claude = nodeAt(transcripts.children, 'claude'); + assert.equal(claude.bytes.value, sizes.recent + sizes.aged); + const project = nodeAt(claude.children, '-repos-keel'); + assert.equal(project.kind, 'project'); + assert.equal(project.attribution, 'path', 'a claude transcript names its project in the path'); + assert.equal(project.path, path.join(claudeProjects, '-repos-keel')); + assert.equal(project.bytes.value, sizes.recent + sizes.aged); + assert.deepEqual(project.children.map((leaf) => leaf.key).sort(), ['aged.jsonl', 'recent.jsonl']); + assert.equal(nodeAt(project.children, 'recent.jsonl').bytes.value, sizes.recent); + + // A codex rollout path is dated, not project-scoped: unattributable, stated. + const codex = nodeAt(transcripts.children, 'codex'); + const codexRoot = nodeAt(codex.children, 'codex-transcripts'); + assert.equal(codexRoot.attribution, 'none'); + assert.equal(codexRoot.bytes.value, sizes.rollout); + + const total = sizes.recent + sizes.aged + sizes.rollout + sizes.index; + assert.equal(result.totals.bytes.value, total); + assert.equal(result.totals.files.value, 4); + assert.equal(result.totals.bytes.partial, false); + assert.equal(result.complete, true); + + // Every category always appears, so a missing slice never reads as a gap. + assert.deepEqual( + result.categories.map((node) => node.key).sort(), + ['kit-caches', 'learning-stores', 'ledgers-and-logs', 'transcripts'], + ); + const empty = nodeAt(result.categories, 'ledgers-and-logs'); + assert.equal(empty.bytes.status, MEASURED); + assert.equal(empty.bytes.value, 0, 'a category with no roots really is zero'); +}); + +test('storage reports "nowhere to look" as unknown, not as a zero-byte category', (t) => { + const { root, now, roots } = storageFixture(t); + const withCatalog = collectStorage({ + roots, projects: [], now: () => now, detectWorktrees: false, fsImpl: fixtureFs(root), + }); + assert.equal(nodeAt(withCatalog.categories, 'learning-stores').bytes.value, 0); + + const noCatalog = collectStorage({ + roots, projects: null, now: () => now, detectWorktrees: false, fsImpl: fixtureFs(root), + }); + const learning = nodeAt(noCatalog.categories, 'learning-stores'); + assert.equal(learning.bytes.status, UNKNOWN); + assert.equal(learning.bytes.value, null); + assert.match(learning.bytes.reason, /no project catalog/); +}); + +test('a capped child list folds its remainder in, so the parent total still adds up', (t) => { + const root = fixture(t, 'storage-cap'); + const now = Date.now(); + const sessions = path.join(root, 'projects', '-repos-keel'); + let total = 0; + for (let i = 0; i < 5; i++) { + total += write(path.join(sessions, `s${i}.jsonl`), 'x'.repeat(10 + i)); + } + + const result = collectStorage({ + roots: [{ id: 'claude-transcripts', category: 'transcripts', host: 'claude', + label: 'session transcripts', path: path.join(root, 'projects'), layout: 'claude-projects' }], + projects: [], now: () => now, detectWorktrees: false, maxChildren: 2, fsImpl: fixtureFs(root), + }); + + const project = nodeAt(nodeAt(nodeAt(result.categories, 'transcripts').children, 'claude') + .children, '-repos-keel'); + assert.equal(project.bytes.value, total); + assert.equal(project.children.length, 3, 'two kept leaves plus one aggregate remainder'); + const aggregate = project.children.at(-1); + assert.equal(aggregate.kind, 'aggregate'); + assert.equal(aggregate.label, '3 more'); + assert.equal(project.children.reduce((acc, child) => acc + child.bytes.value, 0), total); +}); + +test('the default storage roots describe every category without touching the disk', () => { + const roots = storageModule.defaultStorageRoots({ projects: ['/repos/keel'] }); + assert.deepEqual([...new Set(roots.map((entry) => entry.category))].sort(), + [...storageModule.STORAGE_CATEGORIES].sort()); + assert.equal(new Set(roots.map((entry) => entry.id)).size, roots.length, 'ids are unique'); + assert.equal(roots.every((entry) => entry.path && entry.host && entry.label), true); + assert.equal(roots.every((entry) => ['claude-projects', 'flat-sessions', 'tree'] + .includes(entry.layout)), true); + + // A supplied project contributes its three learning stores, attributed to it. + const learning = roots.filter((entry) => entry.category === 'learning-stores'); + assert.deepEqual(learning.map((entry) => path.basename(entry.path)), + ['.claude-flow', '.agentic-qe', '.swarm']); + assert.equal(learning.every((entry) => entry.projectPath === '/repos/keel'), true); + assert.equal(storageModule.defaultStorageRoots() + .some((entry) => entry.category === 'learning-stores'), false); +}); + +test('an unreadable storage root degrades itself while its host siblings report', (t) => { + const root = fixture(t, 'storage-degrade'); + const now = Date.now(); + const readable = path.join(root, 'claude', 'logs'); + const denied = path.join(root, 'claude', 'debug'); + const size = write(path.join(readable, 'a.log'), 'a'.repeat(64)); + write(path.join(denied, 'b.log'), 'b'.repeat(1024)); + + const result = collectStorage({ + roots: [ + { id: 'claude-logs', category: 'ledgers-and-logs', host: 'claude', label: 'logs', + path: readable, layout: 'tree' }, + { id: 'claude-debug', category: 'ledgers-and-logs', host: 'claude', label: 'debug', + path: denied, layout: 'tree' }, + ], + projects: [], now: () => now, detectWorktrees: false, + fsImpl: fixtureFs(root, { deny: [denied] }), + }); + + const host = nodeAt(nodeAt(result.categories, 'ledgers-and-logs').children, 'claude'); + assert.equal(nodeAt(host.children, 'claude-logs').bytes.value, size); + const broken = nodeAt(host.children, 'claude-debug'); + assert.equal(broken.bytes.status, UNKNOWN); + assert.equal(broken.bytes.value, null); + assert.equal(broken.bytes.reason, 'EACCES'); + // The host total keeps what it saw but says it is a floor. + assert.equal(host.bytes.value, size); + assert.equal(host.bytes.partial, true); + assert.equal(result.complete, false); +}); + +test('storage growth derives from mtime and size alone', (t) => { + const { root, now, roots, sizes } = storageFixture(t); + const result = collectStorage({ + roots, projects: [], now: () => now, detectWorktrees: false, fsImpl: fixtureFs(root), + }); + + assert.equal(result.growth.windowDays, 30); + assert.equal(result.growth.approximate, true); + assert.match(result.growth.basis, /mtime/); + const claude = result.growth.hosts.find((host) => host.host === 'claude'); + // Only the 1-day-old transcript falls inside the window; the 200-day-old one + // contributes nothing, and no file's CONTENT was consulted to decide that. + assert.equal(claude.totalBytes.value, sizes.recent); + assert.equal(claude.days.length, 30); + const yesterday = claude.days.find((day) => day.day === localDay(now - DAY)); + assert.deepEqual({ bytes: yesterday.bytes, files: yesterday.files }, + { bytes: sizes.recent, files: 1 }); + // The 400-day-old kit cache is outside the window entirely, so its host has + // no growth series at all rather than a fabricated flat line. + assert.equal(result.growth.hosts.some((host) => host.host === 'agentic-kit'), false); + + const rebuilt = buildGrowth(new Map(), { asOf: now, growthDays: 7 }); + assert.equal(rebuilt.hosts.length, 0); + assert.equal(rebuilt.windowDays, 7); +}); + +test('reclaimable candidates carry a path and a rationale, and nothing can delete', (t) => { + const { root, now, roots, sizes, claudeProjects } = storageFixture(t); + const result = collectStorage({ + roots, projects: [], now: () => now, detectWorktrees: false, fsImpl: fixtureFs(root), + }); + + const aged = result.reclaimables.find((row) => row.id === 'aged-transcripts:claude'); + assert.equal(aged.kind, 'aged-transcripts'); + assert.equal(aged.path, claudeProjects); + assert.equal(aged.bytes.value, sizes.aged, 'only the aged file is a candidate'); + assert.equal(aged.files.value, 1); + assert.deepEqual(aged.samplePaths, [path.join(claudeProjects, '-repos-keel', 'aged.jsonl')]); + assert.match(aged.rationale, /180d/); + assert.equal(aged.advisory, true); + assert.equal(result.reclaimables.every((row) => row.rationale && row.path), true); + + // Invariant 4: the module is advisory. No export removes anything, and the + // cleanup hint is documentation naming the CLI that already owns removal. + for (const name of Object.keys(storageModule)) { + assert.doesNotMatch(name, /delete|remove|prune|clean|unlink|rm/i, + `${name} must not exist on an advisory module`); + } + assert.equal(storageModule.pruneNpxStale, undefined); +}); + +test('an idle npx cache env is a candidate because the cache is reproducible', (t) => { + const root = fixture(t, 'npx'); + const cache = path.join(root, 'npm-cache'); + const env = path.join(cache, '_npx', 'a1b2c3d4'); + const now = Date.now(); + let total = write(path.join(env, 'package.json'), + JSON.stringify({ dependencies: { 'some-tool': '^1.0.0' } })); + total += write(path.join(env, 'node_modules', 'some-tool', 'index.js'), 'x'.repeat(256)); + for (const file of ['package.json', path.join('node_modules', 'some-tool', 'index.js')]) { + touch(path.join(env, file), now - 120 * DAY); + } + + const before = process.env.npm_config_cache; + t.after(() => { + if (before === undefined) delete process.env.npm_config_cache; + else process.env.npm_config_cache = before; + }); + process.env.npm_config_cache = cache; + + const rows = storageModule.npxReclaimables({ + asOf: now, opts: { ...STORAGE_DEFAULTS }, walk: walkTree, limits: {}, fsImpl: fixtureFs(root), + }); + assert.equal(rows.length, 1); + assert.equal(rows[0].kind, 'stale-npx-env'); + assert.equal(rows[0].path, env); + assert.equal(rows[0].label, 'npx cache env (some-tool)'); + assert.equal(rows[0].bytes.value, total); + assert.match(rows[0].rationale, /untouched for 120d/); + assert.match(rows[0].rationale, /re-fetches on demand/); + assert.match(rows[0].cleanupHint, /ak sync/, 'the hint names the CLI that owns removal'); + assert.equal(rows[0].advisory, true); +}); + +test('worktree candidates state what they know and never guess a dead checkout', (t) => { + const root = fixture(t, 'worktrees'); + const now = Date.now(); + const project = path.join(root, 'repo'); + const admin = path.join(project, '.git', 'worktrees'); + + // A record whose checkout is gone, one whose pointer is unreadable, and one + // whose checkout exists but has been idle past the window. + write(path.join(admin, 'dead', 'gitdir'), `${path.join(root, 'gone', '.git')}\n`); + fs.mkdirSync(path.join(admin, 'broken'), { recursive: true }); + const idle = path.join(root, 'idle-checkout'); + const size = write(path.join(idle, 'file.txt'), 'a'.repeat(48)); + write(path.join(admin, 'idle', 'gitdir'), `${path.join(idle, '.git')}\n`); + touch(path.join(idle, 'file.txt'), now - 200 * DAY); + + const rows = worktreeReclaimables({ + asOf: now, projects: [project], opts: { ...STORAGE_DEFAULTS }, + walk: walkTree, limits: {}, fsImpl: fixtureFs(root), + }); + const byKind = (label) => rows.find((row) => row.label.includes(label)); + + const dead = byKind('orphaned worktree record "dead"'); + assert.match(dead.rationale, /no longer exists/); + assert.equal(dead.cleanupHint, 'git worktree prune'); + + const broken = byKind('"broken" (unverifiable)'); + assert.equal(broken.bytes.status, UNKNOWN, 'a pointer we could not read is not a dead worktree'); + assert.equal(broken.bytes.value, null); + + const stale = byKind('idle worktree "idle"'); + assert.equal(stale.path, idle); + assert.equal(stale.bytes.value, size); + assert.match(stale.rationale, /200d/); + assert.equal(rows.every((row) => row.advisory === true), true); +}); + +// ── projects ────────────────────────────────────────────────────────────────── + +test('git remote URLs parse across git+https, ssh, scp and bare shapes', () => { + for (const raw of [ + 'https://github.com/pacphi/agentic-kit.git', + 'git+https://github.com/pacphi/agentic-kit.git', + 'ssh://git@github.com/pacphi/agentic-kit.git', + 'git@github.com:pacphi/agentic-kit.git', + ]) { + const remote = describeRemote(raw); + assert.equal(remote.status, 'linked', raw); + assert.equal(remote.host, 'github', raw); + assert.equal(remote.slug, 'pacphi/agentic-kit', raw); + assert.equal(remote.webUrl, 'https://github.com/pacphi/agentic-kit', raw); + } + + // A self-hosted forge keeps the scheme it was written with — an http-only + // server is never silently upgraded to https. + const selfHosted = describeRemote('http://git.example.com/team/repo.git'); + assert.equal(selfHosted.webUrl, 'http://git.example.com/team/repo'); + assert.equal(selfHosted.host, 'git.example.com'); + + // Unrecognized shapes are reported unlinked: the URL is never guessed. + for (const raw of [ + 'ssh://git@git.internal:2222/team/repo.git', + '/srv/git/bare-repo.git', + 'git@example.com:repo', + '', + ]) { + const remote = describeRemote(raw); + assert.equal(remote.status, 'unrecognized', raw); + assert.equal(remote.webUrl, null, raw); + } +}); + +test('git config parsing prefers origin and never invents one', () => { + const source = [ + '[core]', '\trepositoryformatversion = 0', + '[remote "upstream"]', '\turl = https://github.com/upstream/repo.git', + '[remote "origin"]', '\turl = git@github.com:pacphi/agentic-kit.git', + '\tfetch = +refs/heads/*:refs/remotes/origin/*', + '[branch "main"]', '\tremote = origin', + ].join('\n'); + assert.deepEqual(parseGitRemote(source), { + name: 'origin', url: 'git@github.com:pacphi/agentic-kit.git', + }); + // No origin, but remotes exist: reporting local-only would be a false negative. + assert.deepEqual(parseGitRemote('[remote "fork"]\n\turl = https://example.com/a/b.git\n'), { + name: 'fork', url: 'https://example.com/a/b.git', + }); + assert.equal(parseGitRemote('[core]\n\tbare = false\n'), null); +}); + +test('a project with no remote is local-only, and an unreadable config is unknown', (t) => { + const root = fixture(t, 'remote'); + const bare = path.join(root, 'no-git'); + fs.mkdirSync(bare, { recursive: true }); + assert.equal(projectRemote(bare, { fsImpl: fixtureFs(root) }).status, 'local-only'); + + const noRemote = path.join(root, 'no-remote'); + write(path.join(noRemote, '.git', 'config'), '[core]\n\tbare = false\n'); + const local = projectRemote(noRemote, { fsImpl: fixtureFs(root) }); + assert.equal(local.status, 'local-only'); + assert.equal(local.webUrl, null); + assert.equal(local.reason, null); + + // Unreadable is NOT local-only: absence of evidence is stated as such. + const denied = projectRemote(noRemote, { + fsImpl: { + ...fs, + readFileSync: () => { throw Object.assign(new Error('denied'), { code: 'EACCES' }); }, + }, + }); + assert.equal(denied.status, 'unknown'); + assert.equal(denied.reason, 'EACCES'); +}); + +/** A project fixture with source, vendored, binary and overhead trees. */ +function projectFixture(t) { + const root = fixture(t, 'project'); + const project = path.join(root, 'repo'); + const tree = { + 'src/a.mjs': 'a\nb\nc', + 'src/b.ts': 'x\ny\n', + 'docs/guide.md': 'one\n', + 'vendor/v.js': 'vendored\nlines\n', + 'pnpm-lock.yaml': 'lockfile: true\n', + 'data.bin': 'no recognized extension\n', + }; + const sizes = {}; + for (const [rel, content] of Object.entries(tree)) { + sizes[rel] = write(path.join(project, ...rel.split('/')), content); + } + sizes['binary.js'] = write(path.join(project, 'binary.js'), Buffer.from('let x = 1;\n\0\n')); + const gitBytes = write(path.join(project, '.git', 'config'), + '[remote "origin"]\n\turl = https://github.com/pacphi/agentic-kit.git\n'); + const moduleBytes = write(path.join(project, 'node_modules', 'pkg', 'index.js'), 'module\n') + + write(path.join(project, 'node_modules', 'pkg', 'node_modules', 'dep', 'i.js'), 'nested\n'); + + const treeBytes = Object.values(sizes).reduce((acc, size) => acc + size, 0); + return { root, project, sizes, treeBytes, gitBytes, moduleBytes }; +} + +test('project LOC buckets by extension and excludes vendored, binary and overhead files', (t) => { + const { root, project } = projectFixture(t); + const loc = countLines(project, { fsImpl: fixtureFs(root) }); + + assert.deepEqual(loc.byLanguage, { javascript: 3, typescript: 2, markdown: 1 }); + assert.equal(loc.total.value, 6); + assert.equal(loc.total.status, MEASURED); + assert.equal(loc.total.partial, false, 'a deliberate exclusion is not a failed measurement'); + assert.equal(loc.files, 3); + assert.equal(loc.skipped, 1, 'the NUL-bearing .js file is skipped, not counted as empty'); + assert.equal(loc.approximate, true); + assert.equal(loc.complete, true); + for (const excluded of ['node_modules/', 'vendor/', 'pnpm-lock.yaml']) { + assert.ok(loc.exclusions.includes(excluded), `${excluded} must be stated alongside the figure`); + } + assert.deepEqual(loc.exclusions, LOC_EXCLUSIONS); +}); + +test('project git and node_modules bytes stay separate from the working tree', (t) => { + const { root, project, treeBytes, gitBytes, moduleBytes } = projectFixture(t); + const row = measureProject({ path: project, label: 'repo', source: 'fixture' }, + { fsImpl: fixtureFs(root) }); + + assert.equal(row.treeBytes.value, treeBytes); + assert.equal(row.gitBytes.value, gitBytes); + assert.equal(row.nodeModulesBytes.value, moduleBytes); + assert.equal(row.totalBytes.value, treeBytes + gitBytes + moduleBytes); + assert.deepEqual(row.treeExclusions, ['.git', 'node_modules']); + assert.equal(row.presence, 'present'); + assert.equal(row.complete, true); + assert.equal(row.remote.webUrl, 'https://github.com/pacphi/agentic-kit'); + assert.equal(row.lastActivity.status, MEASURED); + + // The top-most node_modules is the only root: a nested copy is already its + // bytes, so the roots partition rather than overlap. + assert.deepEqual(nodeModulesRoots(project, { fsImpl: fixtureFs(root) }), + [path.join(project, 'node_modules')]); + + // A project with no node_modules at all is a measured zero, not an unknown. + const bare = path.join(root, 'bare'); + write(path.join(bare, 'index.mjs'), 'x\n'); + const bareRow = measureProject({ path: bare, label: 'bare' }, { fsImpl: fixtureFs(root) }); + assert.equal(bareRow.nodeModulesBytes.status, MEASURED); + assert.equal(bareRow.nodeModulesBytes.value, 0); + assert.equal(bareRow.gitBytes.value, 0, 'a missing .git holds a real zero bytes'); +}); + +test('a project path that vanished is unknown everywhere, never a zero-byte project', (t) => { + const root = fixture(t, 'project-missing'); + const row = measureProject({ path: path.join(root, 'gone'), label: 'gone' }, + { fsImpl: fixtureFs(root) }); + assert.equal(row.presence, 'absent'); + for (const figure of [row.treeBytes, row.gitBytes, row.nodeModulesBytes, row.totalBytes, + row.lastActivity, row.loc.total]) { + assert.equal(figure.status, UNKNOWN); + assert.equal(figure.value, null); + } + assert.equal(row.remote.status, 'unknown', 'a vanished path is never reported local-only'); + assert.equal(row.complete, false); +}); + +test('the projects section measures the supplied catalog and reports discovery failure', (t) => { + const { root, project } = projectFixture(t); + const seen = []; + const result = collectProjects({ + projects: [{ path: project, label: 'repo' }, + { path: path.join(root, 'second'), label: 'second' }], + loc: false, limit: 1, fsImpl: fixtureFs(root), + onProgress: (payload) => seen.push(payload.phase), + }); + assert.equal(result.count.value, 2); + assert.equal(result.scanned, 1); + assert.equal(result.truncated, true); + assert.equal(result.locMeasured, false); + assert.equal(result.projects[0].loc.total.status, UNKNOWN, 'LOC not run is unknown, not zero'); + assert.ok(seen.includes('done')); + + const failed = collectProjects({ + discover: () => { throw Object.assign(new Error('nope'), { code: 'EACCES' }); }, + fsImpl: fixtureFs(root), + }); + assert.equal(failed.count.status, UNKNOWN); + assert.equal(failed.count.value, null, 'a failed discovery is not a machine with no projects'); + assert.equal(failed.complete, false); +}); + +// ── catalog ─────────────────────────────────────────────────────────────────── + +/** Host catalog surfaces across claude, codex and opencode, with deliberate + * overlap so dedup and the presence matrix have something to prove. */ +function catalogFixture(t) { + const root = fixture(t, 'catalog'); + const claudeRoot = path.join(root, 'claude'); + const codexRoot = path.join(root, 'codex'); + const opencodeRoot = path.join(root, 'opencode'); + + // The same skill on two hosts, spelled differently: identity is the + // normalized name, so this is ONE deployed skill present twice. + write(path.join(claudeRoot, 'skills', 'Deep-Research', 'SKILL.md'), + '---\nname: x\n---\nSECRET BODY\n'); + write(path.join(codexRoot, 'skills', 'deep-research', 'SKILL.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'skills', 'claude-only', 'SKILL.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'agents', 'reviewer.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'agents', 'v3', 'qe-tester.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'agents', 'README.md'), 'SECRET BODY\n'); + write(path.join(opencodeRoot, 'agents', 'reviewer.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'commands', 'reviewer.md'), 'SECRET BODY\n'); + write(path.join(claudeRoot, 'plugins', 'installed_plugins.json'), + JSON.stringify({ plugins: { 'beads@marketplace': [{ installPath: path.join(root, 'p') }] } })); + + const claudeMcpFile = path.join(root, 'claude.json'); + write(claudeMcpFile, JSON.stringify({ mcpServers: { ruflo: {}, lightpanda: {} } })); + const codexConfigFile = path.join(codexRoot, 'config.toml'); + write(codexConfigFile, '[mcp_servers.ruflo]\ncommand = "npx"\n\n[mcp_servers."with.dot"]\n'); + const opencodeConfigFile = path.join(opencodeRoot, 'opencode.json'); + write(opencodeConfigFile, JSON.stringify({ mcp: { ruflo: {} } })); + + return { + root, claudeRoot, codexRoot, opencodeRoot, claudeMcpFile, codexConfigFile, opencodeConfigFile, + }; +} + +test('catalog dedups by normalized name and keeps a per-host presence matrix', (t) => { + const fixtureRoots = catalogFixture(t); + const { root, ...roots } = fixtureRoots; + const result = collectCatalog({ + ...roots, cwd: root, now: () => 1_700_000_000_000, + includePluginSurfaces: false, fsImpl: fixtureFs(root), + }); + + const skills = result.items.filter((item) => item.kind === 'skill'); + assert.deepEqual(skills.map((item) => item.name).sort(), ['Deep-Research', 'claude-only']); + const shared = skills.find((item) => item.name === 'Deep-Research'); + assert.deepEqual(shared.hosts.sort(), ['claude', 'codex'], + 'case and spacing are presentation; the deployed skill is one thing'); + assert.equal(shared.presence.length, 2); + assert.deepEqual(shared.presence.map((entry) => entry.surface).sort(), + ['claude-skills', 'codex-skills']); + + // Kind is part of identity: a `reviewer` agent and a `reviewer` command are + // two different deployed things. + assert.deepEqual( + result.items.filter((item) => item.name === 'reviewer').map((item) => item.kind).sort(), + ['agent', 'command']); + const agents = result.items.filter((item) => item.kind === 'agent').map((item) => item.name); + assert.deepEqual(agents.sort(), ['reviewer', 'v3:qe-tester']); + assert.ok(!agents.includes('README'), 'a README documents the surface, it is not an entry on it'); + + assert.equal(result.counts.skill.value, 2); + assert.equal(result.counts.agent.value, 2); + assert.equal(result.counts.command.value, 1); + assert.equal(result.counts.mcpServer.value, 3, 'ruflo on three hosts is one server'); + + assert.equal(result.perHost.claude.skill.value, 2); + assert.equal(result.perHost.codex.skill.value, 1); + assert.equal(result.perHost.opencode.skill.value, 0); + assert.equal(result.perHost.opencode.agent.value, 1); + assert.equal(result.perHost.codex.agent.value, 0); + assert.equal(result.perHost.codex.mcpServer.value, 2); + assert.equal(result.complete, true); + assert.deepEqual(result.degraded, []); +}); + +test('catalog counting reads names only — item bodies are never opened', (t) => { + const fixtureRoots = catalogFixture(t); + const { root, ...roots } = fixtureRoots; + const reads = []; + const base = fixtureFs(root); + const spy = { + ...base, + readFileSync: (target, options) => { + reads.push(String(target)); + return base.readFileSync(target, options); + }, + }; + + collectCatalog({ + ...roots, cwd: root, now: () => 1_700_000_000_000, + includePluginSurfaces: false, fsImpl: spy, + }); + for (const file of reads) { + assert.doesNotMatch(file, /SKILL\.md$/, `${file} was opened; a name is all the catalog counts`); + assert.doesNotMatch(file, /(agents|commands)[\\/].*\.md$/, `${file} was opened`); + } +}); + +test('an unreadable catalog surface has no count, and the total it feeds is a floor', (t) => { + const fixtureRoots = catalogFixture(t); + const { root, ...roots } = fixtureRoots; + const denied = path.join(roots.codexRoot, 'skills'); + const result = collectCatalog({ + ...roots, cwd: root, now: () => 1_700_000_000_000, includePluginSurfaces: false, + fsImpl: fixtureFs(root, { deny: [denied] }), + }); + + const surface = result.surfaces.find((entry) => entry.id === 'codex-skills'); + assert.equal(surface.status, 'degraded'); + assert.equal(surface.count, null, 'we did not look, so there is no number'); + assert.equal(surface.reason, 'EACCES'); + // The kinds that surface fed stay measured but become floors; unaffected + // kinds keep clean totals. + assert.equal(result.counts.skill.partial, true); + assert.equal(result.counts.agent.partial, false); + assert.equal(result.complete, false); + assert.deepEqual(result.degraded, ['codex-skills']); + + // A surface that simply is not there is a real zero, distinct from the above. + const absent = result.surfaces.find((entry) => entry.id === 'opencode-commands'); + assert.equal(absent.status, 'absent'); + assert.equal(absent.count, 0); +}); + +test('plugin-contributed entries are namespaced to the plugin that carries them', (t) => { + const fixtureRoots = catalogFixture(t); + const { root, ...roots } = fixtureRoots; + // Both layouts in the wild: content at the plugin root, and content under a + // nested `.claude/`. A plugin using one reports the other absent. + write(path.join(root, 'p', 'skills', 'deep-dive', 'SKILL.md'), 'SECRET BODY\n'); + write(path.join(root, 'p', '.claude', 'agents', 'helper.md'), 'SECRET BODY\n'); + write(path.join(root, 'p', '.mcp.json'), JSON.stringify({ mcpServers: { plugmcp: {} } })); + write(path.join(root, 'cp', 'skills', 'grounding', 'SKILL.md'), 'SECRET BODY\n'); + + const result = collectCatalog({ + ...roots, cwd: root, now: () => 1_700_000_000_000, fsImpl: fixtureFs(root), + inspectCodexPlugins: () => ({ + configPresent: true, + plugins: [{ ref: 'ruvnet-brain@store', root: path.join(root, 'cp') }], + }), + }); + + const names = result.items.map((item) => `${item.kind}:${item.name}`); + assert.ok(names.includes('skill:beads:deep-dive'), names.join(' ')); + assert.ok(names.includes('agent:beads:helper')); + assert.ok(names.includes('mcpServer:beads:plugmcp')); + assert.ok(names.includes('skill:ruvnet-brain:grounding')); + // A host's plugin inventory is its enabled refs, kept whole. + assert.deepEqual(result.items.filter((item) => item.kind === 'plugin').map((item) => item.name), + ['beads', 'ruvnet-brain@store']); + assert.equal(result.perHost.codex.skill.value, 2, + 'a plugin skill joins the host that carries it'); + + // An unreadable codex config is codex's problem, not a reason to lose the + // rest of the catalog. + const survived = collectCatalog({ + ...roots, cwd: root, now: () => 1_700_000_000_000, fsImpl: fixtureFs(root), + inspectCodexPlugins: () => { throw new Error('codex config unreadable'); }, + }); + assert.equal(survived.counts.skill.value >= 2, true); +}); + +test('codex MCP table names are read from config.toml without parsing values', () => { + const source = [ + '[mcp_servers.ruflo]', 'command = "npx"', + '[mcp_servers."with.dot"]', + "[mcp_servers.'single']", + '[other.section]', + ].join('\n'); + assert.deepEqual(tomlTableNames(source, 'mcp_servers'), ['ruflo', 'with.dot', 'single']); + assert.deepEqual(tomlTableNames('', 'mcp_servers'), []); +}); diff --git a/tests/kit/footprint-windows.test.mjs b/tests/kit/footprint-windows.test.mjs new file mode 100644 index 0000000..56ed772 --- /dev/null +++ b/tests/kit/footprint-windows.test.mjs @@ -0,0 +1,428 @@ +// The Windows half of the runtime census, which cannot be verified on the +// machines this repo is developed on. Everything below runs the REAL win32 code +// path with CAPTURED PowerShell output fed through an injected runner, so the +// parsers, the host classification, the root-controller de-nesting and the +// degradation contract are all exercised without a Windows host. +// +// The contract these tests exist to pin down: the census is the GUARANTEED +// floor. `Get-CimInstance Win32_Process` gives pid/ppid/start/image/CPU/RSS and +// always answers; the true per-process cwd comes from a P/Invoke walk of the +// PEB that can be refused by AV, execution policy, permissions or a WOW64 +// mismatch. When that probe fails — for ANY reason — every other field must +// still be reported and only the project attribution may be lost. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { + parseWin32Census, parseWin32Commands, parseWin32Cwds, + parseProcessList, parseProcessMetrics, hostFromCommand, surveyHostProcesses, +} from '../../src/lib/live/process-sessions.mjs'; +import { collectRuntimeCensus } from '../../src/lib/footprint/runtime.mjs'; + +const SCRIPT = 'C:\\opt\\agentic-kit\\scripts\\win-process-survey.ps1'; +const WIN_ENV = { SystemRoot: 'C:\\WINDOWS' }; +const POWERSHELL = path.join( + 'C:\\WINDOWS', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'); + +const WIN_NOW = Date.parse('2026-08-06T12:10:00Z'); +const POSIX_START = 'Thu Aug 6 12:00:00 2026'; +const POSIX_NOW = Date.parse(POSIX_START) + 600_000; + +// Captured `-Mode census` output, CRLF exactly as PowerShell emits it, with the +// noise a real machine carries: the idle process, a kernel row, a shell, and a +// line truncated by a transport that a parser must skip rather than half-read. +const CENSUS = [ + '0\t0\t2026-08-06T11:00:00Z\tSystem_Idle_Process\t0\t8192', + '4\t0\t2026-08-06T11:00:00Z\tSystem\t312500\t147456', + '5000\t4\t2026-08-06T11:30:00Z\texplorer.exe\t93750000\t41943040', + '6100\t5000\t2026-08-06T12:00:00Z\tclaude.exe\t30000000\t536870912', + '6110\t6100\t2026-08-06T12:01:00Z\tpowershell.exe\t5000000\t20971520', + '6111\t6110\t2026-08-06T12:02:00Z\tcodex.exe\t1000000\t104857600', + '6200\t5000\t2026-08-06T12:00:00Z\tnode.exe\t12000000\t268435456', + '6201\t6200\t2026-08-06T12:03:00Z\tcodex.exe\t500000\t83886080', + '6300\t5000\t2026-08-06T11:55:00Z\topencode.exe\t36000000\t157286400', + '6400\t5000\t2026-08-06T11:58:00Z\tcodex.exe\t250000\t62914560', + '6500\t5000\t2026-08-06T11:20:00Z\tclaude.exe\t9000000\t402653184', + '7000\t5000\t2026-08-06T11:59:00Z\ttruncated-row', + '', +].join('\r\n'); + +// Captured `-Mode commands`. 6500 belongs to another user; 6400 is codex's MCP +// server, which is a tool of some other session and never a controller. +const COMMANDS = [ + '6100\town\t"C:\\Users\\dev\\AppData\\Local\\Programs\\claude\\claude.exe" --continue', + '6111\town\t"C:\\Users\\dev\\.codex\\bin\\codex.exe" exec review', + '6200\town\tC:\\nodejs\\node.exe C:\\opt\\bin\\codex', + '6201\town\t"C:\\Users\\dev\\.codex\\bin\\codex.exe"', + '6300\town\tC:\\opencode\\opencode.exe', + '6400\town\t"C:\\Users\\dev\\.codex\\bin\\codex.exe" mcp-server', + '6500\tother\t', + '', +].join('\r\n'); + +const CWDS = [ + '6100\tok\tC:\\repos\\keel', + '6200\tok\tC:\\repos\\agentic-kit', + '6300\terr\topen-denied', + '', +].join('\r\n'); + +/** + * Stands in for `powershell.exe -File win-process-survey.ps1`. `cwd` is a + * function so each test can decide how that one fragile mode behaves — return + * captured output, return nothing, or throw — without touching the other modes. + */ +function winRunner({ census = CENSUS, commands = COMMANDS, cwd = () => CWDS } = {}) { + const calls = []; + const runner = async (command, args) => { + calls.push({ command, args }); + const mode = args[args.indexOf('-Mode') + 1]; + if (mode === 'census') return { stdout: census }; + if (mode === 'commands') return { stdout: commands }; + return { stdout: await cwd(args) }; + }; + runner.calls = calls; + runner.pidsFor = (mode) => { + const call = calls.find((entry) => entry.args[entry.args.indexOf('-Mode') + 1] === mode); + const ids = call?.args[call.args.indexOf('-ProcessIds') + 1]; + return ids ? ids.split(',').map(Number) : []; + }; + return runner; +} + +const win32Survey = (options = {}) => surveyHostProcesses({ + platform: 'win32', scriptPath: SCRIPT, env: WIN_ENV, now: WIN_NOW, ...options, +}); + +// ── the pure census parser ──────────────────────────────────────────────────── + +test('the census parser extracts pid, ppid, start, image, CPU and RSS', () => { + const rows = parseWin32Census(CENSUS); + assert.equal(rows.length, 11, 'the truncated row is skipped, not half-read'); + + const claude = rows.find((row) => row.pid === 6100); + assert.deepEqual(claude, { + pid: 6100, ppid: 5000, startedAt: '2026-08-06T12:00:00Z', executable: 'claude.exe', + command: '', cpuMs: 3000, rssBytes: 536870912, + }); + // Win32_Process reports CPU in 100ns units; the parser normalizes once. + assert.equal(rows.find((row) => row.pid === 6200).cpuMs, 1200); + assert.equal(rows.find((row) => row.pid === 4).cpuMs, 31.25); + assert.equal(rows.every((row) => row.command === ''), + true, 'the census projection never asks for a command line'); +}); + +test('the census parser reports an unusable CPU or RSS field as null, never as 0', () => { + const rows = parseWin32Census([ + '6100\t5000\t2026-08-06T12:00:00Z\tclaude.exe\tnot-a-number\talso-not', + '6200\t5000\t2026-08-06T12:00:00Z\tcodex.exe\t0\t0', + ].join('\n')); + assert.equal(rows[0].cpuMs, null); + assert.equal(rows[0].rssBytes, null); + // A real zero survives as a zero: the distinction is the whole point. + assert.equal(rows[1].cpuMs, 0); + assert.equal(rows[1].rssBytes, 0); +}); + +test('the census parser rejects malformed and non-numeric rows', () => { + assert.deepEqual(parseWin32Census(''), []); + assert.deepEqual(parseWin32Census(null), []); + assert.deepEqual(parseWin32Census([ + 'Get-CimInstance : Access is denied.', + 'pid\tppid\tstart\tname\tcpu\trss', + '6100\t5000\t2026-08-06T12:00:00Z\tclaude.exe', + ].join('\r\n')), []); +}); + +test('the command parser separates owned, foreign and unattributable processes', () => { + const { owned, foreign, failed } = parseWin32Commands([ + '6100\town\tclaude.exe --continue', + '6500\tother\t', + '6600\terr\towner-probe-failed', + '6700\terr\t', + 'garbage', + ].join('\r\n')); + assert.equal(owned.get(6100), 'claude.exe --continue'); + assert.equal(foreign.has(6500), true); + // "someone else's process" and "we could not ask" are different facts, and + // only the first is safe to treat as deliberately excluded. + assert.equal(failed.get(6600), 'owner-probe-failed'); + assert.equal(failed.get(6700), 'owner-probe-failed'); + assert.equal(foreign.has(6600), false); +}); + +test('the cwd parser keeps per-pid successes and per-pid failure reasons apart', () => { + const { found, failures } = parseWin32Cwds([ + '6100\tok\tC:\\repos\\keel', + '6200\terr\twow64-mismatch', + '6300\tok\t', + '6400\terr\t', + ].join('\r\n')); + assert.deepEqual([...found], [[6100, 'C:\\repos\\keel']]); + assert.equal(failures.get(6200), 'wow64-mismatch'); + assert.equal(failures.get(6300), 'cwd-probe-failed', + 'an empty ok path is not a working directory'); + assert.equal(failures.get(6400), 'cwd-probe-failed'); +}); + +// ── classification and de-nesting ───────────────────────────────────────────── + +test('host classification reads a Windows image name the same way it reads a POSIX one', () => { + assert.equal(hostFromCommand('"C:\\Users\\dev\\claude.exe" --continue', 'claude.exe'), 'claude'); + assert.equal(hostFromCommand('C:\\opencode\\opencode.exe', 'opencode.exe'), 'opencode'); + assert.equal(hostFromCommand('C:\\nodejs\\node.exe C:\\opt\\bin\\codex', 'node.exe'), 'codex'); + // The MCP server is a tool of some other session, never a controller. + assert.equal( + hostFromCommand('"C:\\Users\\dev\\.codex\\bin\\codex.exe" mcp-server', 'codex.exe'), null); + assert.equal(hostFromCommand('C:\\tools\\watcher.exe codex', 'watcher.exe'), null); + assert.equal(hostFromCommand('C:\\nodejs\\node.exe C:\\tools\\watch.js', 'node.exe'), null); + // Same commands, POSIX spelling: identical verdicts, which is what makes the + // win32 path testable off Windows at all. + assert.equal(hostFromCommand('/usr/local/bin/claude --continue', 'claude'), 'claude'); + assert.equal(hostFromCommand('node /opt/bin/codex', 'node'), 'codex'); + assert.equal(hostFromCommand('codex mcp-server', 'codex'), null); +}); + +test('the win32 census de-nests to root controllers exactly as the POSIX survey does', async () => { + const runner = winRunner(); + const census = await win32Survey({ execFileImpl: runner }); + + assert.deepEqual(census.processes.map((entry) => ({ host: entry.host, pid: entry.pid })), [ + { host: 'claude', pid: 6100 }, + { host: 'codex', pid: 6200 }, + { host: 'opencode', pid: 6300 }, + ]); + // 6111 (codex under a shell under claude) and 6201 (codex under its own node + // launcher) stay part of their parent's execution graph; 6400 is an MCP + // server; 6500 belongs to another user and had its image name cleared, so + // classification could never promote it. + assert.equal(census.childProcessCount, 3); + assert.equal(census.surveyedProcessCount, 11); + assert.equal(census.platform, 'win32'); + assert.equal(census.observedAt, new Date(WIN_NOW).toISOString()); +}); + +test('the win32 and POSIX surveys agree on the same process topology', async () => { + const win = await win32Survey({ execFileImpl: winRunner() }); + + // The same machine as seen through `ps` + `lsof`. 6500 is absent because the + // POSIX survey scopes to the current uid at the source, where the Windows one + // must filter after the fact with GetOwner. + const processRows = parseProcessList([ + `6100 5000 ${POSIX_START} /usr/local/bin/claude claude --continue`, + `6110 6100 ${POSIX_START} /bin/zsh /bin/zsh -c work`, + `6111 6110 ${POSIX_START} /usr/local/bin/codex codex exec review`, + `6200 5000 ${POSIX_START} node node /opt/bin/codex`, + `6201 6200 ${POSIX_START} /opt/vendor/codex /opt/vendor/codex`, + `6300 5000 ${POSIX_START} /usr/local/bin/opencode opencode`, + `6400 5000 ${POSIX_START} /usr/local/bin/codex codex mcp-server`, + ].join('\n')); + const posix = await surveyHostProcesses({ + platform: 'darwin', processRows, now: POSIX_NOW, + cwdByPid: new Map([[6100, '/repos/keel'], [6200, '/repos/agentic-kit']]), + metricsByPid: parseProcessMetrics([ + '6100 0.5 524288', '6200 0.2 262144', '6300 0.4 153600', + ].join('\n')), + }); + + const shape = (survey) => survey.processes.map((entry) => ({ + host: entry.host, pid: entry.pid, cpuPercent: entry.cpuPercent, rssBytes: entry.rssBytes, + })); + // Different platforms, different sources, one meaning per column: CPU time + // over process lifetime as a percentage of one core, and resident bytes. + assert.deepEqual(shape(win), shape(posix)); + assert.deepEqual(shape(win), [ + { host: 'claude', pid: 6100, cpuPercent: 0.5, rssBytes: 536870912 }, + { host: 'codex', pid: 6200, cpuPercent: 0.2, rssBytes: 268435456 }, + { host: 'opencode', pid: 6300, cpuPercent: 0.4, rssBytes: 157286400 }, + ]); + assert.equal(win.childProcessCount, posix.childProcessCount); + assert.deepEqual(win.processes.map((entry) => entry.uptimeMs), [600_000, 600_000, 900_000]); + assert.deepEqual(posix.processes.map((entry) => entry.uptimeMs), [600_000, 600_000, 600_000]); +}); + +// ── the census survives every cwd failure ───────────────────────────────────── + +test('a per-pid cwd refusal costs that row its project and nothing else', async () => { + const census = await win32Survey({ execFileImpl: winRunner() }); + const [claude, , opencode] = census.processes; + + assert.equal(claude.cwd, 'C:\\repos\\keel'); + assert.equal(claude.cwdReason, null); + // OpenProcess was denied for this one. Every measured field survives. + assert.equal(opencode.cwd, null); + assert.equal(opencode.cwdReason, 'open-denied'); + assert.equal(opencode.pid, 6300); + assert.equal(opencode.ppid, 5000); + assert.equal(opencode.startedAt, '2026-08-06T11:55:00Z'); + assert.equal(opencode.uptimeMs, 900_000); + assert.equal(opencode.cpuPercent, 0.4); + assert.equal(opencode.rssBytes, 157286400); +}); + +test('a cwd probe that fails for EVERY reason still yields a complete census', async () => { + // Each of these is a real way the P/Invoke path dies on a locked-down + // machine: the runner blows up (AV killed powershell, execution policy), + // Add-Type could not compile, the type compiled but the reads were refused, + // and the probe produced nothing parseable at all. + const cases = [ + { label: 'runner threw', cwd: () => { throw new Error('powershell was blocked'); }, + reasons: ['cwd-survey-failed', 'cwd-survey-failed', 'cwd-survey-failed'] }, + { label: 'Add-Type refused', + cwd: () => ['6100\terr\tcompile-failed', '6200\terr\tcompile-failed', + '6300\terr\tcompile-failed'].join('\r\n'), + reasons: ['compile-failed', 'compile-failed', 'compile-failed'] }, + { label: 'mixed refusals', + cwd: () => ['6100\terr\twow64-mismatch', '6200\terr\treader-not-64bit', + '6300\terr\tpeb-read-failed'].join('\r\n'), + reasons: ['wow64-mismatch', 'reader-not-64bit', 'peb-read-failed'] }, + { label: 'no output at all', cwd: () => '', + reasons: ['cwd-unavailable', 'cwd-unavailable', 'cwd-unavailable'] }, + ]; + + for (const { label, cwd, reasons } of cases) { + const census = await win32Survey({ execFileImpl: winRunner({ cwd }) }); + assert.equal(census.processes.length, 3, label); + assert.deepEqual(census.processes.map((entry) => entry.cwdReason), reasons, label); + for (const entry of census.processes) { + assert.equal(entry.cwd, null, label); + // The guaranteed floor: everything Get-CimInstance already answered is + // still here. A cwd failure must never fail the census. + assert.equal(Number.isFinite(entry.pid), true, label); + assert.equal(Number.isFinite(entry.ppid), true, label); + assert.equal(Number.isFinite(entry.uptimeMs), true, label); + assert.equal(Number.isFinite(entry.cpuPercent), true, label); + assert.equal(Number.isFinite(entry.rssBytes), true, label); + assert.ok(entry.host, label); + assert.ok(entry.startedAt, label); + } + assert.equal(census.childProcessCount, 3, label); + } +}); + +test('a partial cwd result keeps what it read instead of discarding the batch', async () => { + const runner = winRunner({ + cwd: () => { + throw Object.assign(new Error('timeout'), { stdout: '6200\tok\tC:\\repos\\agentic-kit' }); + }, + }); + const census = await win32Survey({ execFileImpl: runner }); + assert.deepEqual(census.processes.map((entry) => entry.cwd), + [null, 'C:\\repos\\agentic-kit', null]); + assert.equal(census.processes[0].cwdReason, 'cwd-unavailable'); +}); + +test('a Windows cwd is judged absolute by Windows rules, wherever the test runs', async () => { + const census = await win32Survey({ + execFileImpl: winRunner({ cwd: () => ['6100\tok\tC:\\repos\\keel', '6200\tok\trelative\\path', + '6300\tok\t\\\\server\\share\\repo'].join('\r\n') }), + }); + assert.deepEqual(census.processes.map((entry) => entry.cwd), + ['C:\\repos\\keel', null, '\\\\server\\share\\repo']); + assert.equal(census.processes[1].cwdReason, 'cwd-unavailable'); +}); + +// ── invocation shape ────────────────────────────────────────────────────────── + +test('the survey script is invoked like ps is: absolute path, numeric pids only', async () => { + const runner = winRunner(); + await win32Survey({ execFileImpl: runner }); + + assert.equal(runner.calls.length, 3); + for (const call of runner.calls) { + assert.equal(call.command, POWERSHELL, 'PATH shadowing must not redirect the survey'); + assert.deepEqual(call.args.slice(0, 7), [ + '-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', SCRIPT, + ]); + } + assert.equal(runner.calls[0].args.includes('-ProcessIds'), false, + 'the census covers every process and takes no caller-supplied input'); + // argv is the sensitive field, so it is requested only for pids whose image + // name could actually be a controller or its Node launcher. + assert.deepEqual(runner.pidsFor('commands'), [6100, 6111, 6200, 6201, 6300, 6400, 6500]); + assert.deepEqual(runner.pidsFor('cwd'), [6100, 6200, 6300]); +}); + +test('an unbelievable empty census degrades instead of reporting an idle machine', async () => { + await assert.rejects(win32Survey({ execFileImpl: winRunner({ census: '' }) }), + (error) => error.code === 'ERR_RUNTIME_PROCESS_SURVEY'); + await assert.rejects(win32Survey({ execFileImpl: winRunner({ census: 'Access is denied.' }) }), + (error) => error.code === 'ERR_RUNTIME_PROCESS_SURVEY'); +}); + +test('no host-shaped process means no command-line probe and no cwd probe at all', async () => { + const runner = winRunner({ + census: ['5000\t4\t2026-08-06T11:30:00Z\texplorer.exe\t0\t41943040', + '5001\t5000\t2026-08-06T11:31:00Z\tnotepad.exe\t0\t8388608'].join('\r\n'), + }); + const census = await win32Survey({ execFileImpl: runner }); + assert.deepEqual(census.processes, []); + assert.equal(census.surveyedProcessCount, 2); + assert.equal(runner.calls.length, 1, 'nothing sensitive is read when nothing could be a host'); +}); + +// ── the Runtime section's rendering of all this ─────────────────────────────── + +test('a blocked Windows cwd probe renders as an honest not-attributable project', async () => { + const runner = winRunner({ + cwd: () => ['6100\tok\tC:\\repos\\keel', '6200\terr\tcompile-failed', + '6300\terr\twow64-mismatch'].join('\r\n'), + }); + const census = await collectRuntimeCensus({ + platform: 'win32', + surveyImpl: (options) => surveyHostProcesses({ + ...options, execFileImpl: runner, scriptPath: SCRIPT, env: WIN_ENV, + }), + listDaemonsImpl: async () => [], + osImpl: { totalmem: () => 34359738368, freemem: () => 8589934592, cpus: () => new Array(10) }, + now: WIN_NOW, + }); + + const rows = census.processes.value; + assert.equal(census.ephemeral, true); + assert.equal(rows.length, 3); + + const [claude, codex, opencode] = rows; + assert.equal(claude.project.status, 'measured'); + assert.equal(claude.project.value.path, 'C:\\repos\\keel'); + assert.equal(claude.cwdReason, null); + + // Both of these keep every measured figure and lose only the attribution, + // each with the cause named rather than a blank cell or a guess. + assert.equal(codex.project.status, 'unknown'); + assert.equal(codex.project.value, null); + assert.equal(codex.cwdReason, 'compile-failed'); + assert.match(codex.project.reason, /not attributable on Windows/); + assert.match(codex.project.reason, /could not be built/); + assert.match(opencode.project.reason, /32-bit process, 64-bit probe/); + + for (const row of rows) { + assert.equal(row.rssBytes.status, 'measured'); + assert.equal(row.cpuPercent.status, 'measured'); + assert.equal(row.uptimeMs.status, 'measured'); + } + assert.equal(census.totals.rssBytes.value, 536870912 + 268435456 + 157286400); + assert.equal(census.totals.rssBytes.partial, false); + assert.equal(census.totals.processCount.value, 3); + assert.equal(census.childProcessCount.value, 3); + assert.equal(census.machine.physicalMemoryBytes.value, 34359738368); +}); + +test('a Windows survey that cannot run at all leaves the machine facts standing', async () => { + const census = await collectRuntimeCensus({ + platform: 'win32', + surveyImpl: () => { + throw Object.assign(new Error('nope'), { code: 'ERR_RUNTIME_PROCESS_SURVEY' }); + }, + listDaemonsImpl: async () => [], + osImpl: { totalmem: () => 34359738368, freemem: () => 8589934592, cpus: () => new Array(10) }, + now: WIN_NOW, + }); + assert.equal(census.processes.status, 'unknown'); + assert.equal(census.processes.value, null); + assert.equal(census.totals.rssBytes.status, 'unknown'); + assert.equal(census.totals.processCount.value, null, 'a failed survey is not zero processes'); + assert.match(census.processes.reason, /ERR_RUNTIME_PROCESS_SURVEY/); + assert.equal(census.machine.cpuCount.value, 10); + assert.equal(census.daemons.count.value, 0, 'no daemons running really is zero'); +}); diff --git a/tests/kit/win-process-survey-live.test.mjs b/tests/kit/win-process-survey-live.test.mjs new file mode 100644 index 0000000..17e9f79 --- /dev/null +++ b/tests/kit/win-process-survey-live.test.mjs @@ -0,0 +1,124 @@ +// win-process-survey-live.test.mjs — the ONLY test that actually executes the +// Windows survey. Everything in footprint-windows.test.mjs injects a fake +// runner and feeds the parsers captured fixture text, so those tests pass +// identically on macOS and prove nothing about PowerShell, the .ps1, or the +// P/Invoke probe. This file closes that gap: on win32 it spawns the real +// `powershell -File ` and checks the result against ground +// truth we already hold — THIS process's own pid and cwd. +// +// It is skipped everywhere else, so it is inert locally and only earns its keep +// on the `windows-latest` leg already in the CI matrix (.github/workflows/ci.yml). +// If it ever fails there, one of these is broken for real Windows users: +// packaging (the .ps1 not shipping), execution policy, Get-CimInstance, the +// output contract the JS parsers depend on, or the PEB walk. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const onWindows = process.platform === 'win32'; +const SCRIPT = fileURLToPath( + new URL('../../src/lib/live/win-process-survey.ps1', import.meta.url)); + +/** Invoke the real script exactly as process-sessions.mjs does. */ +async function runScript(mode, processIds) { + const args = [ + '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', + '-File', SCRIPT, '-Mode', mode, + ]; + if (processIds) args.push('-ProcessIds', processIds); + const { stdout } = await execFileAsync('powershell.exe', args, { + encoding: 'utf8', timeout: 60_000, maxBuffer: 8 * 1024 * 1024, + }); + return stdout; +} + +// This one assertion is platform-independent on purpose: a packaging regression +// that drops the .ps1 must fail the suite on every developer's machine, not +// only on the Windows CI leg. This is the exact bug that shipped once already — +// the script lived under scripts/, which package.json's `files` never included, +// so npm-installed Windows users got no census at all. +test('the Windows survey script is colocated under src/ so packaging ships it', () => { + assert.ok(fs.existsSync(SCRIPT), `expected the survey script at ${SCRIPT}`); + const files = JSON.parse( + fs.readFileSync(path.join(path.dirname(SCRIPT), '..', '..', '..', 'package.json'), 'utf8'), + ).files ?? []; + assert.ok( + files.some((entry) => entry === 'src/' || entry.startsWith('src/')), + 'package.json `files` must ship src/, which is what carries the .ps1 into the tarball', + ); + assert.ok( + SCRIPT.split(path.sep).includes('src'), + 'the script must live under src/ — a runtime asset outside the packaged tree never reaches users', + ); +}); + +test('census mode reports THIS process with a real RSS', { skip: !onWindows && 'win32 only' }, async () => { + const stdout = await runScript('census'); + const rows = stdout.split('\n').map((line) => line.trim()).filter(Boolean) + .map((line) => line.split('\t')); + + assert.ok(rows.length > 0, 'the census returned no rows at all'); + for (const row of rows) { + assert.equal(row.length, 6, `each row is pid/ppid/created/name/cpu/rss, got: ${row.join('|')}`); + } + + // Ground truth: we are a live process, so we must be in our own census. + const self = rows.find((row) => Number(row[0]) === process.pid); + assert.ok(self, `this process (pid ${process.pid}) was missing from the census`); + assert.ok(/^node/i.test(self[3]), `expected a node image name, got ${self[3]}`); + assert.ok(Number(self[5]) > 0, 'a live process must report a non-zero working set'); + assert.ok(Number.isFinite(Number(self[4])), 'CPU time must parse as a number'); + + // The floor must never leak argv — that is where a pasted prompt or token lives. + assert.ok(!stdout.includes('--test'), 'census output must not carry command lines'); +}); + +test('cwd mode resolves THIS process to its real directory, or fails honestly', + { skip: !onWindows && 'win32 only' }, async () => { + const stdout = await runScript('cwd', String(process.pid)); + const row = stdout.split('\n').map((line) => line.trim()).filter(Boolean) + .map((line) => line.split('\t')) + .find((cells) => Number(cells[0]) === process.pid); + + assert.ok(row, `cwd mode returned nothing for pid ${process.pid}`); + const [, status, value] = row; + assert.ok(status === 'ok' || status === 'err', + `status must be ok|err so the caller can branch, got ${status}`); + + if (status === 'ok') { + // The strongest available check: we KNOW our own cwd, so a wrong PEB walk + // cannot hide behind a plausible-looking path. + assert.equal( + path.resolve(value).toLowerCase(), + path.resolve(process.cwd()).toLowerCase(), + 'the PEB walk returned a directory that is not actually this process\'s cwd', + ); + } else { + // Honest degradation is a PASS: AV, policy, or bitness may legitimately + // block the probe. What must never happen is a fabricated path. + assert.match(value, /^[a-z0-9-]+$/, + `a failure must be a terse machine-readable reason, got ${value}`); + assert.ok(!value.includes(path.sep), 'a failure reason must never look like a path'); + } + }); + +test('a cwd probe failure never takes the census down with it', + { skip: !onWindows && 'win32 only' }, async () => { + // Pid 0 is the System Idle Process: never openable, so this drives the + // documented failure path rather than simulating it. + const stdout = await runScript('cwd', '0'); + const rows = stdout.split('\n').map((line) => line.trim()).filter(Boolean); + for (const line of rows) { + const cells = line.split('\t'); + assert.equal(cells[1], 'err', 'an unopenable pid must report err, not a path'); + } + // And the guaranteed floor still works immediately afterward. + const census = await runScript('census'); + assert.ok(census.split('\n').filter((line) => line.trim()).length > 0, + 'the census must still work after a failed cwd probe'); + }); From 6054ec1338da69a9b02dab207c2dbb73d1e7d134 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 16:31:54 -0700 Subject: [PATCH 07/19] feat(system): GET /api/system, snapshot persistence, and the ak system CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cheap tier (census + known-file stats + snapshot carry-forward, TTL-cached) on every read; deep tier explicit and single-flight so concurrent refreshes attach to the in-flight scan. The deep result persists with an asOf; a missing or corrupt snapshot reads as "never measured", never as zeros. Rescan is manual only — nothing scans on dashboard open. The payload deliberately carries absolute paths, unlike /api/live's leaf-only reduction, because in this domain the path is the answer; file contents are never read, so nothing sensitive can travel with them. dashboard.test.cjs's self-contained assertion is replaced with a shared assertSelfContained() helper. The old regex conflated "no external fetch" with "no https string" and so failed on About's curated link pills; the replacement pins the invariant to the directory itself — every external URL must be one about-directory.mjs declares — and still bans external script/stylesheet/img. That is strictly stronger, and the browser suite independently asserts the run requests nothing off the loopback origin. --- bin/agentic-kit.mjs | 4 + src/commands/system.mjs | 346 +++++++++++++++++++++++++++++++++ src/lib/dashboard-server.mjs | 83 +++++++- tests/dashboard.test.cjs | 359 ++++++++++++++++++++++++++++++++++- 4 files changed, 782 insertions(+), 10 deletions(-) create mode 100644 src/commands/system.mjs diff --git a/bin/agentic-kit.mjs b/bin/agentic-kit.mjs index a38a3cd..0df99c9 100755 --- a/bin/agentic-kit.mjs +++ b/bin/agentic-kit.mjs @@ -22,6 +22,8 @@ const PORCELAIN = Object.assign(Object.create(null), { dashboard: () => import('../src/commands/x/dashboard.mjs'), admin: () => import('../src/commands/x/admin.mjs'), usage: () => import('../src/commands/usage.mjs'), + system: () => import('../src/commands/system.mjs'), + about: () => import('../src/commands/about.mjs'), run: () => import('../src/commands/run.mjs'), host: () => import('../src/commands/x/host.mjs'), uninstall: () => import('../src/commands/uninstall.mjs'), @@ -49,6 +51,8 @@ Usage (ak = alias of agentic-kit): ak dashboard open the local web dashboard (localhost; auto-opens browser) [--port N] [--no-open] ak admin maintainer-only telemetry admin (localhost; GitHub/npm egress) [--port N] [--no-open] ak usage inspect/refresh offline provider analytics [status|refresh openrouter] + ak system what this stack occupies on your machine [--deep] [--json] + ak about what agentic-kit installs and configures, and why [--category N] ak run execute a host-neutral activity pipeline [template "task"] [--dry-run] ak host manage agent hosts, routing, and provider bindings [status|pick|refresh|off] ak uninstall leave cleanly [--this-project] [--purge] diff --git a/src/commands/system.mjs b/src/commands/system.mjs new file mode 100644 index 0000000..bfdd474 --- /dev/null +++ b/src/commands/system.mjs @@ -0,0 +1,346 @@ +// ak system — the machine footprint in the terminal (ADR-0025). +// +// The CLI twin of the dashboard's System area, driving the SAME composed +// collector (src/lib/footprint/index.mjs) over the same two tiers. Reading is +// cheap and always safe: the live process census, the individually-known files, +// and whatever the last deep scan persisted, carried forward with THAT scan's +// timestamp. The expensive walk runs only under --deep, only when a human asked +// for it — never on open, never on a nudge (the nudge just says the figures are +// getting old). +// +// Every number on this page is a Measurement, and this file's whole job is to +// render one honestly. A measured zero prints as 0 because it IS zero. An +// unmeasured quantity prints the reason it is missing and NEVER a 0 (ADR-0023, +// machine-footprint invariant 2). A capped or partially-degraded walk prints +// with a `>=` because what it measured is a floor, not a total. +import { heading, info, ok, warn, dim, bold, withProgress } from '../lib/output.mjs'; +import { createSystemCollector } from '../lib/footprint/index.mjs'; +import { UNKNOWN } from '../lib/footprint/walk.mjs'; + +export const options = { + json: { type: 'boolean', default: false }, + deep: { type: 'boolean', default: false }, +}; + +export const help = `ak system — what this stack occupies on your machine + +Reads the cheap tier by default: the live agent-process census, the files that +grow fastest between scans, and the last deep scan's figures carried forward with +the date they were taken. --deep re-walks install trees, storage, the cross-host +catalog, and every discovered project, then persists the result. + +Usage: + ak system [options] + +Options: + --deep re-run the full scan now (minutes on a large machine), then persist it + --json emit the snapshot payload verbatim — the same shape /api/system serves + +Examples: + ak system install totals, runtime census, storage, catalog, projects + ak system --deep re-measure everything, then print it + ak system --json machine-readable snapshot (no scan) + ak system --deep --json re-measure, then emit the fresh snapshot`; + +const UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + +/** Decimal units, matching the System design mock. */ +function fmtBytes(n) { + if (!Number.isFinite(n)) return String(n); + let value = Math.abs(n); + let unit = 0; + while (value >= 1000 && unit < UNITS.length - 1) { value /= 1000; unit += 1; } + const digits = unit === 0 ? 0 : value >= 100 ? 0 : value >= 10 ? 1 : 2; + return `${(n < 0 ? -value : value).toFixed(digits)} ${UNITS[unit]}`; +} + +const fmtCount = (n) => (Number.isFinite(n) ? n.toLocaleString('en-US') : String(n)); +const fmtPercent = (n) => (Number.isFinite(n) ? `${n.toFixed(1)}%` : String(n)); + +function fmtDuration(ms) { + if (!Number.isFinite(ms) || ms < 0) return String(ms); + const s = Math.floor(ms / 1000); + if (s < 60) return `${s}s`; + if (s < 3600) return `${Math.floor(s / 60)}m`; + if (s < 86_400) return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`; + return `${Math.floor(s / 86_400)}d ${Math.floor((s % 86_400) / 3600)}h`; +} + +const fmtAgo = (at, now) => (Number.isFinite(at) ? `${fmtDuration(now - at)} ago` : String(at)); +const fmtStamp = (at) => (Number.isFinite(at) + ? new Date(at).toISOString().replace('T', ' ').slice(0, 16) + : 'unknown'); + +/** + * Render a Measurement as a line value. An unknown carries its reason with it — + * that reason is the whole point of the type, and dropping it here would leave + * a bare "unknown" indistinguishable from a rendering bug. + */ +function meas(value, fmt = fmtCount) { + if (!value || typeof value !== 'object') return dim('unknown — no measurement reported'); + if (value.status === UNKNOWN) return dim(`unknown — ${value.reason}`); + return `${value.partial ? '>= ' : ''}${fmt(value.value)}`; +} + +/** + * Table cells cannot carry a reason: column widths are computed on the raw text, + * so a long reason (or an ANSI escape) would wreck the alignment. The sink keeps + * every distinct reason it swallowed and the caller prints them under the table, + * so "unknown" in a cell is still traceable to why. + */ +function reasonSink() { + const seen = new Set(); + return { + cell(value, fmt = fmtCount) { + if (!value || typeof value !== 'object') { seen.add('no measurement reported'); return 'unknown'; } + if (value.status === UNKNOWN) { seen.add(value.reason); return 'unknown'; } + return `${value.partial ? '>= ' : ''}${fmt(value.value)}`; + }, + report(indent = ' ') { + for (const reason of seen) console.log(`${indent}${dim(`unknown: ${reason}`)}`); + }, + }; +} + +/** Left-aligned columns; the last column is never padded so lines do not carry + * trailing whitespace. Cells must be plain text (see reasonSink). */ +function table(headers, rows, indent = ' ') { + if (!rows.length) return; + const widths = headers.map((header, i) => Math.max( + header.length, ...rows.map((row) => String(row[i] ?? '').length), + )); + const line = (cells) => cells + .map((cell, i) => (i === cells.length - 1 ? String(cell ?? '') : String(cell ?? '').padEnd(widths[i]))) + .join(' ') + .trimEnd(); + console.log(`${indent}${dim(line(headers))}`); + for (const row of rows) console.log(`${indent}${line(row)}`); +} + +const field = (label, value) => console.log(` ${dim(label.padEnd(16))}${value}`); + +/** A deep section's provenance. Every figure inside it was taken at one instant + * and every one of them reads back as `carried-forward` (the section is always + * served from the persisted snapshot, even microseconds after --deep wrote it), + * so the scan date is stated ONCE here instead of on all several hundred lines + * — invariant 3 satisfied without burying the figures. */ +function deepHeading(name, section) { + const asOf = section?.asOf; + return heading(`${name}${Number.isFinite(asOf) ? dim(` — measured ${fmtStamp(asOf)}`) : ''}`); +} + +function renderSummary(snapshot) { + const { snapshot: snap, runtime, knownFiles } = snapshot; + heading('ak system — machine footprint'); + field('platform', snapshot.platform); + field('census', `${runtime?.ephemeral ? 'live' : 'reported'} · ${snapshot.generatedAt}`); + const present = knownFiles?.nodes?.filter((node) => node.presence === 'present').length ?? 0; + field('known files', `${present}/${knownFiles?.nodes?.length ?? 0} present`); + if (!snap?.present) { + field('deep scan', dim(`never run — ${snap?.reason ?? 'no snapshot'}`)); + } else { + const missing = snap.completeness?.missing ?? []; + field('deep scan', `${fmtStamp(snap.asOf)} (${fmtDuration(snap.ageMs)} ago)` + + (missing.length ? dim(` · ${missing.join(', ')} not measured`) : '')); + } +} + +function renderInstall(install) { + deepHeading('Install', install); + if (!install) { + info(dim('not measured yet — run: ak system --deep')); + return; + } + field('tools present', meas(install.totals?.toolsPresent)); + field('install size', meas(install.totals?.installBytes, fmtBytes)); + field('shared caches', meas(install.totals?.cacheBytes, fmtBytes)); + field('native addons', meas(install.totals?.nativeAddons)); + field('disk', `${meas(install.disk?.freeBytes, fmtBytes)} free of ` + + `${meas(install.disk?.totalBytes, fmtBytes)}`); + if (install.globalRootReason) field('npm root', dim(install.globalRootReason)); + + const sink = reasonSink(); + const rows = (install.tools ?? []).map((tool) => [ + tool.label, + tool.present ? (tool.version ? `v${tool.version}` : 'present') : 'absent', + tool.installMethod, + sink.cell(tool.bytes, fmtBytes), + tool.rootReason ?? '', + ]); + console.log(''); + table(['TOOL', 'VERSION', 'METHOD', 'SIZE', 'NOTE'], rows); + sink.report(); + const dupes = install.duplicateNatives ?? []; + if (dupes.length) info(`${dupes.length} native module(s) compiled into more than one tree`); +} + +function renderRuntime(runtime) { + heading(`Runtime${dim(' — live, never persisted')}`); + if (!runtime) { + info(dim('no census reported')); + return; + } + field('processes', meas(runtime.totals?.processCount)); + field('memory (RSS)', meas(runtime.totals?.rssBytes, fmtBytes)); + field('cpu', meas(runtime.totals?.cpuPercent, fmtPercent)); + field('daemons', `${meas(runtime.daemons?.count)} running · ${meas(runtime.daemons?.staleCount)} stale`); + field('machine', `${meas(runtime.machine?.physicalMemoryBytes, fmtBytes)} memory · ` + + `${meas(runtime.machine?.freeMemoryBytes, fmtBytes)} free · ${meas(runtime.machine?.cpuCount)} cores`); + + const census = runtime.processes; + if (census?.status === UNKNOWN) { + console.log(` ${dim(`process table unavailable — ${census.reason}`)}`); + return; + } + const rows = census?.value ?? []; + if (!rows.length) { + console.log(` ${dim('no agent processes are running')}`); + return; + } + const sink = reasonSink(); + console.log(''); + table(['HOST', 'PID', 'CPU', 'RSS', 'UPTIME', 'PROJECT'], rows.map((row) => [ + row.host, + String(row.pid), + sink.cell(row.cpuPercent, fmtPercent), + sink.cell(row.rssBytes, fmtBytes), + sink.cell(row.uptimeMs, fmtDuration), + row.project?.status === UNKNOWN ? 'unattributed' : (row.project?.value?.label ?? 'unattributed'), + ])); + sink.report(); + // `project` degrades per process (a cwd the platform will not disclose); its + // reason lives on the row, not in the numeric sink above. + for (const reason of new Set(rows.filter((row) => row.project?.status === UNKNOWN) + .map((row) => row.project.reason))) { + console.log(` ${dim(`unattributed: ${reason}`)}`); + } +} + +function renderStorage(storage) { + deepHeading('Storage', storage); + if (!storage) { + info(dim('not measured yet — run: ak system --deep')); + return; + } + field('total', `${meas(storage.totals?.bytes, fmtBytes)} · ${meas(storage.totals?.files)} files`); + + const sink = reasonSink(); + console.log(''); + table(['CATEGORY', 'SIZE', 'FILES'], (storage.categories ?? []).map((category) => [ + category.label, + sink.cell(category.bytes, fmtBytes), + sink.cell(category.files), + ])); + sink.report(); + + const reclaimables = storage.reclaimables ?? []; + if (reclaimables.length) { + const advisory = reasonSink(); + console.log(''); + console.log(` ${dim('reclaimable (advisory only — ak system removes nothing)')}`); + table(['CANDIDATE', 'SIZE', 'CLEANUP'], reclaimables.map((row) => [ + row.label, advisory.cell(row.bytes, fmtBytes), row.cleanupHint ?? '—', + ])); + advisory.report(); + } +} + +function renderCatalog(catalog) { + deepHeading('Catalog', catalog); + if (!catalog) { + info(dim('not measured yet — run: ak system --deep')); + return; + } + const kinds = catalog.kinds ?? []; + field('deduplicated', kinds.map((kind) => `${meas(catalog.counts?.[kind])} ${kind}`).join(' · ')); + + const sink = reasonSink(); + console.log(''); + table(['HOST', ...kinds.map((kind) => kind.toUpperCase())], (catalog.hosts ?? []).map((host) => [ + host, ...kinds.map((kind) => sink.cell(catalog.perHost?.[host]?.[kind])), + ])); + sink.report(); + if (catalog.degraded?.length) info(dim(`unreadable surfaces: ${catalog.degraded.join(', ')}`)); + if (catalog.truncated?.length) { + info(dim(`capped surfaces (counts are floors): ${catalog.truncated.join(', ')}`)); + } +} + +function renderProjects(projects, now) { + deepHeading('Projects', projects); + if (!projects) { + info(dim('not measured yet — run: ak system --deep')); + return; + } + field('discovered', `${meas(projects.count)}${projects.truncated ? dim(' · list truncated') : ''}`); + if (!projects.locMeasured) field('lines of code', dim('not measured in this scan')); + + const sink = reasonSink(); + console.log(''); + table(['PROJECT', 'SIZE', 'LOC', 'LAST ACTIVE'], (projects.projects ?? []).map((project) => [ + project.label, + sink.cell(project.totalBytes, fmtBytes), + sink.cell(project.loc?.total), + sink.cell(project.lastActivity, (at) => fmtAgo(at, now)), + ])); + sink.report(); +} + +/** The staleness nudge — the ONLY thing that ever suggests a rescan. It never + * triggers one: a deep walk costs minutes, so it stays a human's decision. */ +function renderNudge(snap) { + if (!snap?.present) { + info(`no deep scan on this machine yet — run: ${bold('ak system --deep')}`); + } else if (snap.stale) { + warn(`deep figures are ${fmtDuration(snap.ageMs)} old — refresh with: ${bold('ak system --deep')}`); + } +} + +/** + * @param {{ flags: Record, + * deps?: { collector?: ReturnType, + * cwd?: string, now?: () => number } }} input + */ +export async function run({ flags, deps = {} }) { + const collector = deps.collector ?? createSystemCollector({ cwd: deps.cwd ?? process.cwd() }); + const now = deps.now ?? Date.now; + + let scan = null; + if (flags.deep) { + // refreshDeep never rejects by contract; it reports failure in its result so + // a partly-completed scan still yields the sections that DID finish. + scan = await withProgress('deep scan', () => collector.refreshDeep(), { + // The ticker owns a stdout line via \r-rewrites — it must never interleave + // with a --json payload. + tty: process.stdout.isTTY && !flags.json, + }); + } + + // The collector assembles the deep sections by spread, so the inferred return + // type cannot name install/storage/catalog/projects. The wire shape is the + // contract (ADR-0025); widening here reads it without restating it. + /** @type {Record} */ + const snapshot = await collector.read(); + + if (flags.json) { + console.log(JSON.stringify(snapshot, null, 2)); + return scan && !scan.ok ? 1 : 0; + } + + renderSummary(snapshot); + if (scan) { + if (scan.ok) ok(`deep scan complete in ${fmtDuration(snapshot.scan?.durationMs ?? 0)}`); + else warn(`deep scan failed: ${scan.error}`); + // A scan that measured everything but could not write the file is still a + // successful measurement — say which of the two happened. + if (scan.ok && scan.error) warn(scan.error); + } + renderInstall(snapshot.install); + renderRuntime(snapshot.runtime); + renderStorage(snapshot.storage); + renderCatalog(snapshot.catalog); + renderProjects(snapshot.projects, now()); + console.log(''); + renderNudge(snapshot.snapshot); + return scan && !scan.ok ? 1 : 0; +} diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index 58565e5..304f97a 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -31,6 +31,12 @@ // GET /api/usage → the usage Aggregate MINUS sessions[] (ADR-0009) // GET /api/sessions → the session list, filtered + paginated // GET /api/session/:id → one transcript, secrets masked SERVER-side +// GET /api/system → the machine-footprint payload (ADR-0025): the cheap +// tier (runtime census + known-file stats, TTL-cached +// ~60s) merged with the last persisted deep snapshot, +// carried forward with ITS asOf. `?refresh=deep` starts +// or attaches to the single-flight deep scan and returns +// immediately with progress state. // // The status rows are gathered by SHELLING OUT to the installed CLI // (`node bin/agentic-kit.mjs status --json`) so we never duplicate status.mjs's @@ -501,6 +507,21 @@ function sendTranscriptJson(res, status, payload) { res.end(JSON.stringify(payload)); } +/** Lazily bind the machine-footprint collector (ADR-0025), for the same reason + * lazyUsage is lazy: the System area pulls in five walkers plus the runtime + * survey, and a panel that never opens that tab must not pay for them. One + * instance per dashboard server — it owns the cheap tier's TTL cache and the + * deep scan's single-flight slot, and a second instance would defeat both. */ +function lazySystem(systemOptions = {}) { + let instancePromise; + return async () => { + instancePromise ||= import('./footprint/index.mjs').then(({ createSystemCollector }) => ( + createSystemCollector(systemOptions) + )); + return instancePromise; + }; +} + /** Load the collector only when Live is requested. This keeps dashboard startup * cheap and permits tests/embedders to inject a source without touching real * transcript stores. */ @@ -525,7 +546,8 @@ function lazyLive(liveOptions = {}) { * any|Promise)|any, * intelClientBuffer?: number, intelMaxClients?: number, * discoverProjects?: () => Array<{ path: string, label: string, source?: string }>, - * machineWideIntel?: (projects: Array) => any }} [opts] + * machineWideIntel?: (projects: Array) => any, + * system?: any, systemOptions?: any }} [opts] * @returns {Promise<{ url: string, urlWithToken: string, port: number, token: string, close: () => Promise }>} */ export function startDashboard({ @@ -534,7 +556,7 @@ export function startDashboard({ liveOptions = {}, liveIdleMs = 30_000, transcripts, transcriptOptions = {}, transcriptClientBuffer = 64, transcriptMaxClients = 16, intelWatch, intelClientBuffer = 256, intelMaxClients = 32, - discoverProjects, machineWideIntel, + discoverProjects, machineWideIntel, system, systemOptions = {}, } = {}) { const provide = fetchStatus || shellOutStatus(cwd); const usageApi = usage || lazyUsage(); @@ -542,6 +564,22 @@ export function startDashboard({ // real ~/.config through this route. Lazy for the same reason lazyUsage is. const provideLimits = limits || (async () => (await import('./quota.mjs')).readLimits()); const provideLive = typeof live === 'function' ? live : live ? async () => live : lazyLive(liveOptions); + // Same injection contract as `live`: a function is called to produce the + // collector, a value is reused verbatim (tests hand over a fake so the route + // never walks the real machine), and the default is the lazy real one. The + // collector must expose read/refreshDeep — checked at use, not here, so an + // unopened System tab still costs nothing. + const provideSystem = typeof system === 'function' + ? system : system ? async () => system : lazySystem({ cwd, ...systemOptions }); + let systemPromise; + const getSystem = async () => { + const collector = await (systemPromise ||= Promise.resolve().then(provideSystem)); + if (!collector || typeof collector.read !== 'function' + || typeof collector.refreshDeep !== 'function') { + throw new TypeError('system collector must implement read and refreshDeep'); + } + return collector; + }; let transcriptServicePromise; const provideTranscripts = typeof transcripts === 'function' ? transcripts : transcripts ? async () => transcripts : async () => { @@ -1236,6 +1274,47 @@ export function startDashboard({ return; } + // ── System / machine footprint (ADR-0025). Lazy, like Usage above. ────── + // + // DELIBERATE DIVERGENCE from publicLivePayload, documented in ADR-0025 §7 + // and the machine-footprint DDD's delivery section: this payload carries + // ABSOLUTE PATHS and is NOT run through publicLivePayload's leaf-only + // reduction. That reduction exists because a path is incidental provenance + // on a *session* payload; here the path IS the answer — a storage + // breakdown that hides where the bytes live answers nothing. The exposure + // is bounded by what the collectors structurally cannot do: file CONTENTS + // are never read (stat metadata, directory entries, and manifest names + // only), so no transcript, prompt, or tool payload can reach this route to + // leak. Delivery protections are otherwise identical to every route above + // — loopback bind, per-session token auth, no-store, nosniff, zero egress. + if (url === '/api/system') { + try { + const collector = await getSystem(); + // ORDER IS LOAD-BEARING: assemble the payload BEFORE starting a scan. + // The deep collectors are synchronous, so the first phase occupies the + // event loop the moment it gets a turn — and `read()` awaits, which + // hands it that turn. Starting first therefore made the *initiating* + // request wait out the phase it had just kicked off (measured: 9s), + // which is precisely the hang the progress state exists to avoid. + const payload = await collector.read(); + if (query.get('refresh') === 'deep') { + // Start-or-attach and answer NOW. The collector's single flight means + // a second refresh joins the running scan rather than racing it, and + // it never rejects — the catch guards an injected collector that does + // not honour that contract, so a bad one cannot take the process down + // with an unhandled rejection. + Promise.resolve(collector.refreshDeep()).catch(() => {}); + // The payload predates the start by microseconds; re-stamp the live + // scan block so this response reads "running", not "idle". + if (typeof collector.scanState === 'function') payload.scan = collector.scanState(); + } + sendJson(res, 200, payload); + } catch (e) { + sendJson(res, 503, { error: 'system footprint unavailable', reason: String(e && e.message || e) }); + } + return; + } + if (url === '/api/sessions') { try { const agg = await usageApi.readIndex({ days: clampDays(query.get('days')) }); diff --git a/tests/dashboard.test.cjs b/tests/dashboard.test.cjs index bfd3c7e..4d3d8a9 100644 --- a/tests/dashboard.test.cjs +++ b/tests/dashboard.test.cjs @@ -33,6 +33,32 @@ function contains(hay, needle) { assert(String(hay).includes(needle), `expected output to contain ${JSON.stringify(needle)}`); } +// "Self-contained" means the page FETCHES nothing off-origin. It does not mean +// no https string may appear: the About directory ships curated GitHub/npm/docs +// anchors the user clicks in their own browser, which is a stated design point +// (docs/ddd/component-directory.md §6 — "Links are outbound and user-initiated; +// the kit stays offline"). So the invariant is pinned to the directory itself — +// every external URL baked into the page must be one the directory declares. +// A CDN script, webfont, tracking beacon, or any other new external host still +// fails here, because its URL is not in that set. +let directoryUrls = null; +async function assertSelfContained(body) { + if (!directoryUrls) { + const { directoryEntries } = await import('../src/lib/dashboard/about-directory.mjs'); + directoryUrls = new Set(); + for (const e of directoryEntries()) for (const l of e.links || []) directoryUrls.add(l.url); + } + const unexpected = (body.match(/https?:\/\/[^"'`\s\\)]+/g) || []) + .filter((u) => !/^https?:\/\/127\.0\.0\.1/.test(u) && !/w3\.org/.test(u)) + .filter((u) => !directoryUrls.has(u)); + assert(unexpected.length === 0, + 'page must not reference external hosts beyond the About directory anchors; found: ' + + unexpected.slice(0, 5).join(', ')); + assert(!/]+stylesheet/i.test(body), 'no external stylesheet links'); + assert(!/]+src=/i.test(body), 'no external script src'); + assert(!/]+src=["']https?:/i.test(body), 'no external image src'); +} + function mkFixture(files) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dash-test-')); for (const [rel, data] of Object.entries(files)) { @@ -162,10 +188,7 @@ async function main() { await test('GET / is self-contained — no external fetches', async () => { const r = await get(url); - assert(!/https?:\/\/(?!127\.0\.0\.1)/.test(r.body.replace(/https?:\/\/[^"'\s]*w3\.org/g, '')), - 'page must not reference external http(s) hosts'); - assert(!/]+stylesheet/i.test(r.body), 'no external stylesheet links'); - assert(!/]+src=/i.test(r.body), 'no external script src'); + await assertSelfContained(r.body); }); // Security review Finding 4: the dashboard — a larger inline-script @@ -643,6 +666,328 @@ async function main() { await limitsSrv.close(); } + // ── /api/system (ADR-0025): the machine-footprint payload ──────────────── + // + // Driven through the REAL composed collector with every collaborator + // injected: the fs impl refuses every call, the persisted snapshot is handed + // over as a value, and no deep collector walks anything. A hand-rolled fake + // collector would only prove the fake — the two-tier merge and the + // single-flight slot the route depends on live in the composition, not in + // the route, so the route must be exercised against the real one. + const { createSystemCollector } = await import( + 'file://' + path.join(ROOT, 'src', 'lib', 'footprint', 'index.mjs')); + const { measured, unknown } = await import( + 'file://' + path.join(ROOT, 'src', 'lib', 'footprint', 'walk.mjs')); + + const SCAN_ASOF = 1785000000000; // when the persisted deep scan ran + const SYS_NOW = SCAN_ASOF + 3_600_000; // an hour later: fresh, nowhere near the 7d nudge + const SYS_HOME = path.resolve(path.sep, 'home', 'tester'); + const SNAPSHOT_FILE = path.join(SYS_HOME, '.config', 'agentic-kit', 'footprint-snapshot.json'); + + // Every fs entry point the collector can reach, refused. A test that stats a + // real home directory measures the developer's laptop, not the code. + const sysEnoent = () => { const e = new Error('ENOENT'); e.code = 'ENOENT'; throw e; }; + const NO_FS = { + lstatSync: sysEnoent, statSync: sysEnoent, readdirSync: sysEnoent, + readFileSync: sysEnoent, writeFileSync: sysEnoent, mkdirSync: sysEnoent, + renameSync: sysEnoent, copyFileSync: sysEnoent, unlinkSync: sysEnoent, + existsSync: () => false, + }; + + // What a previous deep scan left behind. Absolute paths are the POINT of + // this payload (ADR-0025 §7), so the fixture carries them everywhere. + const deepSections = () => ({ + install: { + asOf: SCAN_ASOF, complete: true, + nodes: [{ + id: 'ak-config', path: path.join(SYS_HOME, '.config', 'agentic-kit'), + bytes: measured(4096, { asOf: SCAN_ASOF }), + }], + }, + storage: { + asOf: SCAN_ASOF, complete: true, + total: measured(123456, { asOf: SCAN_ASOF }), + roots: [{ + id: 'claude', path: path.join(SYS_HOME, '.claude'), + bytes: measured(123456, { asOf: SCAN_ASOF }), + }], + }, + catalog: { asOf: SCAN_ASOF, complete: true, agents: [], skills: [], commands: [] }, + projects: { + asOf: SCAN_ASOF, complete: true, + rows: [{ + path: path.join(SYS_HOME, 'src', 'demo'), label: 'demo', + loc: measured(900, { asOf: SCAN_ASOF }), + }], + }, + }); + + const scannedSnapshot = () => ({ + present: true, asOf: SCAN_ASOF, writtenAt: new Date(SCAN_ASOF).toISOString(), + schemaVersion: 1, completeness: { complete: true, sections: {}, missing: [] }, + sections: deepSections(), reason: null, file: SNAPSHOT_FILE, + }); + + // The one honest shape for "nothing has ever been deep-scanned here". + const neverScanned = () => ({ + present: false, asOf: null, writtenAt: null, schemaVersion: null, + completeness: null, sections: null, file: SNAPSHOT_FILE, + reason: 'no deep scan has been run on this machine', + }); + + const runtimeCensus = () => ({ + observedAt: new Date(SYS_NOW).toISOString(), + platform: 'darwin', + ephemeral: true, + processes: measured([{ + host: 'claude', pid: 4242, startedAt: new Date(SCAN_ASOF).toISOString(), cwdReason: null, + uptimeMs: measured(3_600_000), cpuPercent: measured(1.5), rssBytes: measured(512_000), + project: measured({ path: path.join(SYS_HOME, 'src', 'demo'), label: 'demo', key: 'demo' }), + }]), + childProcessCount: measured(1), + totals: { + processCount: measured(1), rssBytes: measured(512_000), cpuPercent: measured(1.5), + }, + daemons: { + count: measured(0), staleCount: measured(0), ttlSecs: 43200, + oldestAgeSecs: unknown('no daemons are running'), + budget: unknown('ruflo exposes no local budget state this collector can read'), + entries: [], + }, + machine: { + physicalMemoryBytes: measured(16_000_000_000), + freeMemoryBytes: measured(8_000_000_000), + cpuCount: measured(10), + }, + }); + + // `calls` counts every collector invocation, so single-flight is asserted by + // COUNT rather than by timing — the only assertion that cannot pass by luck. + function systemFixture({ snapshot = scannedSnapshot(), collectors = {} } = {}) { + const calls = { runtime: 0, install: 0, storage: 0, catalog: 0, projects: 0, persist: 0 }; + const tally = (key, fn) => (...args) => { calls[key]++; return fn(...args); }; + const sections = deepSections(); + const collector = createSystemCollector({ + now: () => SYS_NOW, + fsImpl: NO_FS, + cwd: fixture, + loadConfig: () => ({}), + discoverProjects: () => [{ path: path.join(SYS_HOME, 'src', 'demo'), label: 'demo' }], + readSnapshotImpl: () => snapshot, + writeSnapshotImpl: () => { + calls.persist++; + return { ok: true, file: SNAPSHOT_FILE, asOf: SCAN_ASOF, error: null }; + }, + collectors: { + runtime: tally('runtime', collectors.runtime || (async () => runtimeCensus())), + install: tally('install', collectors.install || (() => sections.install)), + storage: tally('storage', collectors.storage || (() => sections.storage)), + catalog: tally('catalog', collectors.catalog || (() => sections.catalog)), + projects: tally('projects', collectors.projects || (() => sections.projects)), + }, + }); + return { calls, collector }; + } + + const sysFx = systemFixture(); + const sysSrv = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: sysFx.collector, + }); + try { + await test('GET /api/system with a MISSING or WRONG token → 401, and no collector runs', async () => { + const refused = [ + await get(sysSrv.url + 'api/system'), + await get(sysSrv.url + 'api/system', 'not-the-token'), + ]; + for (const r of refused) { + assert(r.status === 401, 'expected 401, got ' + r.status); + const j = JSON.parse(r.body); + assert(typeof j.error === 'string', '401 must still be JSON with a reason'); + for (const key of ['runtime', 'knownFiles', 'storage', 'projects', 'snapshot', 'scan']) { + assert(!(key in j), '401 body must carry no data field, found ' + key); + } + } + // Auth is refused BEFORE the collector is touched — an unauthenticated + // caller must not be able to make this machine do work. + assert(sysFx.calls.runtime === 0, + 'the collector ran for an unauthenticated request (' + sysFx.calls.runtime + ' census call(s))'); + }); + + await test('GET /api/system → 200 no-store: the cheap tier merged with the persisted snapshot', async () => { + const r = await get(sysSrv.url + 'api/system', sysSrv.token); + assert(r.status === 200, 'expected 200, got ' + r.status); + assert(r.headers['cache-control'] === 'no-store', 'a machine census must never be cached'); + assert(r.headers['x-content-type-options'] === 'nosniff', 'nosniff on this route too'); + contains(r.headers['content-type'] || '', 'application/json'); + const j = JSON.parse(r.body); + assert(j.generatedAt === new Date(SYS_NOW).toISOString(), 'generatedAt must be this request'); + assert(j.cheapTier.asOf === SYS_NOW, 'the cheap tier is stamped now, got ' + j.cheapTier.asOf); + assert(j.runtime.totals.processCount.value === 1, 'the census rides on every read'); + assert(Array.isArray(j.knownFiles.nodes) && j.knownFiles.nodes.length > 0, + 'the known-file stats are part of the cheap tier'); + for (const key of ['install', 'storage', 'catalog', 'projects']) { + assert(j[key], 'the persisted section ' + key + ' must merge into the payload'); + } + // Invariant 3: a figure measured an hour ago arrives carrying THAT asOf, + // re-stamped carried-forward so no renderer can read it as current. + assert(j.storage.total.status === 'carried-forward' && j.storage.total.asOf === SCAN_ASOF, + 'deep figures must be carried forward with the scan asOf, got ' + JSON.stringify(j.storage.total)); + assert(j.snapshot.present === true && j.snapshot.asOf === SCAN_ASOF, 'snapshot asOf must survive'); + assert(j.snapshot.ageMs === 3_600_000 && j.snapshot.stale === false, + 'an hour-old scan is fresh — the nudge is a 7-day rule'); + assert(j.scan.running === false && j.scan.phase === 'idle', + 'reading must never start a scan (manual-rescan-only)'); + assert(sysFx.calls.install === 0, 'a plain read must not run a deep collector'); + }); + + await test('the System payload deliberately carries absolute paths — and no transcript content', async () => { + const r = await get(sysSrv.url + 'api/system', sysSrv.token); + const j = JSON.parse(r.body); + // DELIBERATE DIVERGENCE from /api/live's leaf-only reduction (ADR-0025 + // §7): a storage breakdown that hides where the bytes live answers + // nothing, so here the absolute path IS the answer and must survive. + contains(r.body, path.join(SYS_HOME, '.claude')); + contains(r.body, path.join(SYS_HOME, 'src', 'demo')); + assert(j.storage.roots[0].path === path.join(SYS_HOME, '.claude'), + 'a storage root must keep its absolute path'); + assert(j.runtime.processes.value[0].project.value.path === path.join(SYS_HOME, 'src', 'demo'), + 'a process must keep the absolute cwd it was attributed to'); + assert(j.knownFiles.nodes.every((n) => path.isAbsolute(n.path)), + 'every known file is named by absolute path'); + // The exposure is bounded by what the collectors structurally cannot do: + // they stat, they never read file contents. Nothing message-shaped may + // appear anywhere in this payload, at any depth. + const keys = new Set(); + (function walkKeys(v) { + if (Array.isArray(v)) { v.forEach(walkKeys); return; } + if (!v || typeof v !== 'object') return; + for (const [k, item] of Object.entries(v)) { keys.add(k); walkKeys(item); } + })(j); + for (const banned of ['turns', 'messages', 'message', 'content', 'text', 'prompt', 'transcript']) { + assert(!keys.has(banned), 'the System payload must carry no transcript field, found ' + banned); + } + }); + } finally { + await sysSrv.close(); + } + + await test('?refresh=deep is single-flight — two concurrent refreshes share one scan', async () => { + let release; + const gate = new Promise((resolve) => { release = resolve; }); + // Gate the CHEAP tier, not the deep one. Both requests then resume from the + // same promise in one microtask drain, and runDeep's first act is a + // setImmediate — so the second request PROVABLY reaches refreshDeep() while + // the first still holds the slot. Racing two bare HTTP requests would be + // testing the scheduler, not the single-flight rule. + const fx = systemFixture({ collectors: { runtime: async () => { await gate; return runtimeCensus(); } } }); + const srv = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: fx.collector, + }); + try { + const both = Promise.all([ + get(srv.url + 'api/system?refresh=deep', srv.token), + get(srv.url + 'api/system?refresh=deep', srv.token), + ]); + await eventually(() => fx.calls.runtime === 2, 'both refreshes must reach the collector'); + release(); + const [a, b] = await both; + assert(a.status === 200 && b.status === 200, 'both refreshes must answer 200'); + const scanA = JSON.parse(a.body).scan; + assert(scanA.running === true && scanA.phase !== 'idle', + 'a refresh must report the scan it started, got ' + JSON.stringify(scanA)); + await eventually(() => fx.calls.persist === 1, 'the shared scan must run to completion'); + assert(fx.calls.install === 1 && fx.calls.storage === 1 + && fx.calls.catalog === 1 && fx.calls.projects === 1, + 'the deep collectors ran twice — the single-flight slot did not hold: ' + JSON.stringify(fx.calls)); + assert(fx.calls.persist === 1, 'a shared scan must write exactly one snapshot'); + } finally { + await srv.close(); + } + }); + + await test('a never-scanned machine reports "never measured", never zeros', async () => { + const fx = systemFixture({ snapshot: neverScanned() }); + const srv = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: fx.collector, + }); + try { + const r = await get(srv.url + 'api/system', srv.token); + assert(r.status === 200, 'expected 200, got ' + r.status); + const j = JSON.parse(r.body); + // ADR-0023 / invariant 2: null with a reason, never an object of zeros — + // there must be no numeric field for a renderer to misread as "0 bytes". + for (const key of ['install', 'storage', 'catalog', 'projects']) { + assert(j[key] === null, key + ' must be null, not zeros: ' + JSON.stringify(j[key])); + } + assert(j.snapshot.present === false && j.snapshot.measured === false, 'snapshot must read unmeasured'); + assert(j.snapshot.asOf === null && j.snapshot.ageMs === null, 'unmeasured has no age'); + assert(j.snapshot.stale === false, 'a machine that never scanned is unmeasured, not stale'); + contains(j.snapshot.reason, 'no deep scan'); + // The other half of the rule: a MEASURED zero stays a real zero. Under + // the refusing fs every known file is genuinely absent, and an absent + // file genuinely holds no bytes. + const absent = j.knownFiles.nodes.find((n) => n.presence === 'absent'); + assert(absent && absent.bytes.status === 'measured' && absent.bytes.value === 0, + 'an absent file is a measured zero, got ' + JSON.stringify(absent && absent.bytes)); + assert(fx.calls.install === 0, 'an unmeasured machine must not auto-scan on open'); + } finally { + await srv.close(); + } + }); + + await test('a throwing collector degrades its own section — the route still answers 200', async () => { + const fx = systemFixture({ + collectors: { runtime: async () => { throw new Error('ps: permission denied'); } }, + }); + const srv = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: fx.collector, + }); + try { + const r = await get(srv.url + 'api/system', srv.token); + assert(r.status === 200, 'one blown section must not take the whole route down, got ' + r.status); + const j = JSON.parse(r.body); + assert(j.runtime.processes === null, 'a failed census must not fabricate an empty process list'); + contains(j.runtime.error, 'permission denied'); + assert(j.storage.total.value === 123456, 'the sections that DID measure must still render'); + assert(j.knownFiles.nodes.length > 0, 'the rest of the cheap tier survives a failed census'); + } finally { + await srv.close(); + } + }); + + await test('a collector that cannot be built at all degrades to 503 with a reason', async () => { + const srv = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: async () => { throw new Error('footprint module unavailable'); }, + }); + try { + const r = await get(srv.url + 'api/system', srv.token); + assert(r.status === 503, 'expected 503, got ' + r.status); + const j = JSON.parse(r.body); + contains(j.reason, 'footprint module unavailable'); + for (const key of ['runtime', 'knownFiles', 'storage', 'snapshot']) { + assert(!(key in j), 'a failed route must fabricate no data, found ' + key); + } + // A collector missing the contract is the same class of failure, and must + // land the same way rather than throwing past the handler. + const bad = await startDashboard({ + port: 0, cwd: fixture, fetchStatus: async () => STUB_STATUS, usage: spyUsage().api, + system: { read: async () => ({}) }, + }); + try { + const r2 = await get(bad.url + 'api/system', bad.token); + assert(r2.status === 503, 'a collector without refreshDeep must be 503, got ' + r2.status); + contains(r2.body, 'refreshDeep'); + } finally { await bad.close(); } + } finally { + await srv.close(); + } + }); + // Masking is not optional: a usage module that cannot mask must fail the // request, never serve an unmasked transcript. const noMask = spyUsage({ maskSecrets: undefined }); @@ -901,9 +1246,7 @@ async function main() { await test('served page is still self-contained after the Usage tab lands', async () => { const r = await get(uiSrv.url); - assert(!/https?:\/\/(?!127\.0\.0\.1)/.test(r.body.replace(/https?:\/\/[^"'\s]*w3\.org/g, '')), - 'page must not reference external http(s) hosts'); - assert(!/]+src=/i.test(r.body), 'no external script src'); + await assertSelfContained(r.body); }); } finally { await uiSrv.close(); @@ -1447,7 +1790,7 @@ async function main() { // is the suite where it matters most — the traversal-guard and credential- // leak tests live here and were the reviewer's cited example of a block // that could silently vanish with the old harness never noticing. - const EXPECTED = 65; + const EXPECTED = 72; if (passed + failed !== EXPECTED) { console.error(`\nPLAN MISMATCH: expected ${EXPECTED} tests, ran ${passed + failed}`); process.exit(1); From bbfb1f346685189149b12666570d0bded87caf02 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Thu, 6 Aug 2026 16:31:54 -0700 Subject: [PATCH 08/19] feat(dashboard): About and System primary areas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About lands leftmost as the reading-order entry point; Overview remains the default landing view, with a dismissible first-run nudge rather than a hijacked view. System adds Summary/Storage/Runtime/Catalog/Projects with the charted treatments from the design mock, a freshness label that nudges once a snapshot goes stale, and honest empty states — "not measured yet" is never rendered as a zero. This makes five primary areas. ADR-0005's "exactly three stable primary areas" assertion is updated to state the new contract, which both ADR-0025 and ADR-0026 record as a deliberate amendment. --- src/lib/dashboard/client.mjs | 1045 +++++++++++++++++++++++++++++++++- src/lib/dashboard/page.mjs | 287 +++++++++- src/lib/dashboard/styles.mjs | 229 ++++++++ tests/ui/dashboard-ui.mjs | 659 ++++++++++++++++++++- 4 files changed, 2193 insertions(+), 27 deletions(-) diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index d8a9814..2a321c5 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -1,4 +1,5 @@ import { CAT, RANK, PREF, esc, catOf, groupRows, rowLine, groupCard, gridHtml, noticeHtml } from './groups.mjs'; +import { directoryEntries } from './about-directory.mjs'; // The classification/grouping/card/notice logic lives in ./groups.mjs (pure — // unit-testable in node without a DOM). Here those exact function sources and @@ -11,12 +12,19 @@ const objLiteral = (o) => `{${Object.entries(o).map(([k, v]) => `${ident(k) ? k const CAT_JS = objLiteral(CAT); const RANK_JS = objLiteral(RANK); const PREF_JS = JSON.stringify(PREF); +// The About area's authored directory, serialized into the bundle rather than +// fetched: it is release-versioned editorial content, not machine state, so it +// ships with the page and needs no endpoint of its own (ADR-0026). Runtime +// facts arrive from the /api/status payload the dashboard already polls and are +// joined in the browser. +const ABOUT_JS = JSON.stringify(directoryEntries()); export const JS = ` (function(){ "use strict"; var root=document.documentElement; var LS="ak-dash-theme", LS_TAB="ak-dash-tab", LS_OVERVIEW="ak-dash-overview-view"; + var LS_SYSTEM="ak-dash-system-view", LS_ABOUT_NUDGE="ak-dash-about-nudge"; // Dashboard-wide session token (ADR-0014). Bootstrap is idempotent and // duplicated from live-view.mjs's copy (separate diff --git a/src/lib/dashboard/styles.mjs b/src/lib/dashboard/styles.mjs index 7eda06e..a7f29e5 100644 --- a/src/lib/dashboard/styles.mjs +++ b/src/lib/dashboard/styles.mjs @@ -885,15 +885,6 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} @media(max-width:600px){.sy-3,.sy-4,.sy-5{grid-column:span 12}} .sy-head{display:flex; align-items:baseline; justify-content:space-between; gap:8px} .sy-head h3{margin:0; font-size:13px; font-weight:650; letter-spacing:-.008em} -.sy-form{ - flex:none; font-family:var(--mono); font-size:10px; color:var(--ink-dim); white-space:nowrap; - border:1px solid var(--line); border-radius:100px; padding:1px 8px; -} -.sy-note{ - margin-top:auto; padding-top:8px; border-top:1px dashed var(--line-2); - font-size:11.5px; color:var(--ink-2); line-height:1.45; -} -.sy-note b{color:var(--ink); font-weight:600} .sy-legend{display:flex; gap:12px; flex-wrap:wrap; font-size:11.5px; color:var(--ink-2)} .sy-legend i{display:inline-block; width:9px; height:9px; border-radius:3px; margin-right:5px; vertical-align:-1px} .sy-legend b{color:var(--ink); font-weight:600} diff --git a/tests/dashboard.test.cjs b/tests/dashboard.test.cjs index 4d3d8a9..42e20e5 100644 --- a/tests/dashboard.test.cjs +++ b/tests/dashboard.test.cjs @@ -847,8 +847,12 @@ async function main() { // DELIBERATE DIVERGENCE from /api/live's leaf-only reduction (ADR-0025 // §7): a storage breakdown that hides where the bytes live answers // nothing, so here the absolute path IS the answer and must survive. - contains(r.body, path.join(SYS_HOME, '.claude')); - contains(r.body, path.join(SYS_HOME, 'src', 'demo')); + // Compare against the JSON-ENCODED form. A Windows absolute path carries + // backslashes, which JSON escapes on the wire, so a raw substring check + // passes on POSIX and fails on Windows for a payload that is correct. + const onWire = (p) => JSON.stringify(p).slice(1, -1); + contains(r.body, onWire(path.join(SYS_HOME, '.claude'))); + contains(r.body, onWire(path.join(SYS_HOME, 'src', 'demo'))); assert(j.storage.roots[0].path === path.join(SYS_HOME, '.claude'), 'a storage root must keep its absolute path'); assert(j.runtime.processes.value[0].project.value.path === path.join(SYS_HOME, 'src', 'demo'), From 230665952ac7ee29bb8caa7b9a0384dbb503c864 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 7 Aug 2026 06:55:46 -0700 Subject: [PATCH 11/19] fix(dashboard): correct foldKnownVersions' JSDoc so tsc --checkJs passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI quality gate went red on the previous commit: the param was annotated Array<{pkg:string}> while the function pushes {pkg, installed, latest, outdated}, so tsc rejected the object literal. The annotation was simply narrower than the array driftReport() and the selfDrift/brain/ruvector folds have always produced. The return type keeps those fields OPTIONAL rather than required, because incoming entries are passed through untouched — promising them as present would be a second wrong annotation in the other direction. --- src/lib/dashboard-server.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index d1ebf67..625a0fe 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -406,8 +406,14 @@ async function cachedHostFacts(now) { * giving the version chip the structured fact it needs. Nothing here parses a * version out of a status row's prose; every value is a structured probe. * Existing entries always win, so this can only ever add. - * @param {Array<{pkg:string}>|null} drift + * @param {Array<{pkg:string, installed?:string|null, latest?:string|null, + * outdated?:boolean}>|null} drift the drift array, in the shape driftReport() + * and the selfDrift/brain/ruvector folds all already emit * @param {{ now?: number, hostFacts?: Record }} [deps] test seam + * @returns {Promise>} the input array plus the folded entries; incoming + * entries are passed through untouched, so their fields stay as optional as + * whichever fold produced them */ export async function foldKnownVersions(drift, { now = Date.now(), hostFacts } = {}) { const out = [...(drift ?? [])]; From 76bdd287b8ef68b2c4b2abcb074fd4a7c0570db0 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 7 Aug 2026 07:05:56 -0700 Subject: [PATCH 12/19] test(ui): defuse the dated fixture corpus, and match the hero's new contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UI suite went from 241/0 to 183/25 with no code change between the runs. Cause: the fixture corpus is pinned to 2026-07-24 and the panel requests a 14-day window, so at 00:00 on 2026-08-07 the whole corpus aged out of its own window. Proof, straight from the index: days=14 -> 0 sessions, days=30 -> 3. One data-fixture check failed and 23 session-view assertions cascaded off it. The kit suites avoid this by pinning `now` (usage-index.test.mjs says so in its header), but this harness drives a REAL server against the real clock, so it cannot. extendedCorpus() already copies the fixtures into a temp dir, so the copy is shifted forward instead — by a WHOLE number of days, which preserves every relative fact the assertions rest on: the 85-minute idle gap separating the three time tiers, the worktree session nested inside another's span, and each turn's local time-of-day for the punchcard's hour buckets. The checked-in fixtures keep their literal dates, because the kit suites pin `now` against exactly those. The remaining failure was a real contract change, not a bomb: the hero no longer counts detections, so asserting it says "unknown" tested behaviour that was deliberately removed. It now asserts the actual contract — the hero states only what ak MANAGES, making no detection claim in either direction, because each card's own chip already carries per-component state and an aggregate could only restate it less precisely. --- tests/ui/dashboard-ui.mjs | 49 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/ui/dashboard-ui.mjs b/tests/ui/dashboard-ui.mjs index 74a520e..f44a932 100644 --- a/tests/ui/dashboard-ui.mjs +++ b/tests/ui/dashboard-ui.mjs @@ -200,9 +200,46 @@ function extendedCorpus() { asst(WT_ID, wtCwd, '2026-07-24T11:40:00.000Z', 'Rebased.'), )); + slideCorpusIntoWindow([claude, codex]); return { claude, codex }; } +/** The fixtures and the sessions written above are all pinned to FIXTURE_EPOCH. + * The kit suites cope with that by pinning `now` (see usage-index.test.mjs's + * header), but this harness drives a REAL server against the real clock and + * the panel requests a 14-day window, so a fixed date is a dated bomb: the + * corpus passed every run until it turned 14 days old, then every + * session-dependent assertion went blind at once against unchanged code. + * + * Shifting the copied corpus keeps it permanently inside the window. The shift + * is a WHOLE NUMBER OF DAYS so every relative fact the assertions actually + * test survives it — the 85-minute idle gap that separates the three time + * tiers, the worktree session nested inside another's span, and each turn's + * local time-of-day, which the punchcard buckets by hour. */ +const FIXTURE_EPOCH = '2026-07-24'; +function slideCorpusIntoWindow(roots) { + const DAY = 86_400_000; + // Land the corpus a few days back: comfortably inside the 14-day default, + // and never in the future, which would read as an unfinished session. + const target = Date.now() - 3 * DAY; + const shiftDays = Math.round((target - Date.parse(`${FIXTURE_EPOCH}T00:00:00.000Z`)) / DAY); + if (shiftDays <= 0) return; // fixtures are already recent enough + const shiftMs = shiftDays * DAY; + const ISO = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z/g; + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) { walk(p); continue; } + if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue; + fs.writeFileSync(p, fs.readFileSync(p, 'utf8').replace(ISO, (stamp) => { + const at = Date.parse(stamp); + return Number.isFinite(at) ? new Date(at + shiftMs).toISOString() : stamp; + })); + } + }; + for (const root of roots) walk(root); +} + // ── views under test ───────────────────────────────────────────────────────── // ORDER IS THE CONTRACT, not an implementation detail: About leads because it // is the orientation surface (ADR-0026) and System trails because it is the @@ -2461,8 +2498,16 @@ async function main() { && degraded.entries.every((entry) => entry.state === 'unknown' && /unknown/i.test(entry.chip) && entry.reason.length > 0), `states were ${JSON.stringify([...new Set(degraded.entries.map((e) => e.state))])}`); - check('the hero says component states are unknown instead of counting detections', - /unknown/i.test(degraded.lede) && !/report as installed/i.test(degraded.lede), + // The hero states only what ak MANAGES — a release fact, true on any machine + // and unaffected by a failed join. It deliberately makes no detection claim + // in either direction: no "N of N installed" tally, and no aggregate + // "states are unknown" either, because the chip on every card already says + // so per-component (asserted directly above) and an aggregate could only + // restate that less precisely. + check('the hero makes no detection claim, so a lost join cannot make it lie', + !/report as installed/i.test(degraded.lede) + && !/\bof \d+\b/.test(degraded.lede) + && /manages/i.test(degraded.lede), `the lede read ${JSON.stringify(degraded.lede.slice(0, 200))}`); const degradedArts = artifactsIn(await visibleText(page, '#panel-about')); check('the degraded About area is free of rendering artifacts', degradedArts.length === 0, From 2387a35f0f999a86e2a24e22487f9de487a59db2 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 7 Aug 2026 09:13:17 -0700 Subject: [PATCH 13/19] feat(system): wire the footprint collectors, fix a hard hang, expand reclaimables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collectors from the previous run were built and tested but never called — index.mjs had no owner, so nothing composed them into the payload. Wiring them up is most of this change; the rest is what wiring them exposed. Now live (verified against GET /api/system, not asserted): - projects: 50 ever seen / 25 on disk / 21 git repos, de-duped across hosts by resolved real path. Was 4, because discovery reused discoverRuvfloProjects() — which requires .claude-flow/neural/ state and answers a different question that Intelligence still depends on, so it is left alone. - consumers: 80 roots, top 20 ranked, grouped by ecosystem. The panel called npx cache the #1 consumer at 6 GB; it is #12. The real leaders were entirely unscanned — Ollama 141 GB, LM Studio 49 GB, Hugging Face 36 GB, npm's _cacache 22 GB. Overlapping roots collapse by path so the list cannot go self-similar (~/.npm never appears beside its own _cacache). - snapshot persists consumers; runtime is still absent, and that allow-list is what structurally enforces the ephemeral-census invariant. - refreshDeep finally takes includeProjectTrees, which the caller was already passing into a zero-arity function. Sticky across rescans; default off, because one repository here is 175 GB and flattens every other row. A HARD HANG, found only because the wider discovery reached it: the deep scan parked forever at 0% CPU inside a Dropbox tree. Cloud providers leave evicted placeholders — stat returns instantly, read blocks in the kernel until the provider materializes the bytes, which never happens while it is signed out. There is no timeout, so the scan never completed and the dashboard served a 17-hour-old snapshot. Worse, reading them silently pulls the file down. Guarded by the allocated-blocks basis the DDD already names (blocks === 0 && size > 0): placeholders are stat-ed, never opened. Zero false positives across 3,667 files here, including sub-2KB files that APFS stores inline. The tree that hung forever now returns in 50ms; a full scan takes 204s. Also: Playwright was invisible on macOS (the scan looked only at the XDG and Windows paths, so 1.86 GB read as a measured zero), and the RuvNet Brain was under-reported by 85% because only kb/ was measured — the other 11 GB is five dated kb.bak snapshots, now broken out rather than silently folded in. Reclaimables grew from two detectors to cover those backups, npm's regenerable cache, orphaned transcripts, and browser downloads — split into two safety tiers that never sum: 'regenerable' for caches a tool refetches on demand, and 'review' for things like mise's 8 node versions, where recommending deletion of a live runtime would be worse than saying nothing. --- docs/DASHBOARD.md | 76 +- docs/adr/0025-machine-footprint-metrics.md | 210 ++++- docs/ddd/machine-footprint.md | 378 +++++++- src/lib/dashboard-server.mjs | 13 +- src/lib/dashboard/client.mjs | 632 ++++++++++++-- src/lib/dashboard/page.mjs | 44 +- src/lib/dashboard/styles.mjs | 153 +++- src/lib/footprint/consumers.mjs | 961 +++++++++++++++++++++ src/lib/footprint/index.mjs | 147 +++- src/lib/footprint/install.mjs | 209 ++++- src/lib/footprint/project-sources.mjs | 448 ++++++++++ src/lib/footprint/projects.mjs | 503 +++++++---- src/lib/footprint/snapshot.mjs | 22 +- src/lib/footprint/stack-detect.mjs | 582 +++++++++++++ src/lib/footprint/stack-registry.mjs | 610 +++++++++++++ src/lib/footprint/storage.mjs | 812 ++++++++++++++++- src/lib/footprint/walk.mjs | 14 +- tests/kit/footprint-projects.test.mjs | 348 ++++++++ tests/kit/footprint-stack.test.mjs | 388 +++++++++ 19 files changed, 6126 insertions(+), 424 deletions(-) create mode 100644 src/lib/footprint/consumers.mjs create mode 100644 src/lib/footprint/project-sources.mjs create mode 100644 src/lib/footprint/stack-detect.mjs create mode 100644 src/lib/footprint/stack-registry.mjs create mode 100644 tests/kit/footprint-projects.test.mjs create mode 100644 tests/kit/footprint-stack.test.mjs diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index 5a594f6..d659721 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -44,11 +44,11 @@ permanent. | Usage | Transcript | `#usage/transcript` | Transcript detail | The selected session's locally retained, server-masked evidence | | Observability | Live | `#observability/live` | Observability · Live | Projects and roots with current presence or fresh meaningful activity | | Observability | History | `#observability/history` | Observability · History | Retained roots that are not currently Live | -| System | Summary | `#system/summary` | Summary | Install size, retained data, live resource use, and deployed inventory in one glance | -| System | Storage | `#system/storage` | Storage | Where the retained bytes are, by category, host, project, and session — plus growth and advisory reclaimables | +| System | Summary | `#system/summary` | Summary | Install size, retained data, live resource use, deployed inventory, and the machine's largest storage consumers in one glance | +| System | Storage | `#system/storage` | Storage | Where the retained bytes are, by category, host, project, and session — plus growth and advisory reclaimables in two safety tiers | | System | Runtime | `#system/runtime` | Runtime | Live host processes, their CPU and memory, background daemons, and machine denominators | | System | Catalog | `#system/catalog` | Catalog | Deduplicated skills, agents, commands, plugins, and MCP servers, with a per-host presence matrix | -| System | Projects | `#system/projects` | Projects | Every known project's approximate lines of code, working-tree, `.git`, and `node_modules` size | +| System | Projects | `#system/projects` | Projects | Projects ever seen across hosts vs still on disk; per on-disk project its approximate lines of code, detected stack, working-tree, `.git`, and `node_modules` size | About is one scrolling page, so its hashes scroll to a section rather than swapping panels; `#about` alone opens the page at the top. @@ -200,6 +200,48 @@ measured, and once a snapshot passes seven days the freshness label turns amber rows are advisory, with their rationale and their path, and there is no delete button anywhere in this area. +### Largest consumers, and the project-trees toggle + +The Summary strip ranks the biggest storage roots on the machine — not just the kit's own. On a +working machine the top of that list is usually local model weights, package caches and toolchain +installs, so the strip covers around fifty known cache roots (Ollama, LM Studio, Hugging Face, +npm/pnpm/yarn/bun, rustup and Cargo, Go, uv and pip, Maven and Gradle, Playwright and Puppeteer, +mise, Homebrew, Docker) alongside the kit's own. **Ranked** and **By ecosystem** re-shape the +same measurement; the ecosystem view is usually the more actionable one, because four Node caches +at 5 GB each is a Node answer, not four unrelated rows. + +Nested roots are counted **once**, at the outermost row. `~/.npm/_cacache` sits inside `~/.npm`, +`~/.cache/huggingface` inside `~/.cache`, the npm global root inside mise's Node install — so a +row inside another row is shown as a *breakdown* of its parent and is left out of the ranking and +the totals. Every parent with breakdowns also gets an "everything else" row, so a breakdown always +adds up to its parent. Roots that do not exist on this machine are listed as absent rather than +ranked at 0 B, and roots that could not be read say so with their reason. + +**Project trees** are excluded by default, and the chip that includes them is a *scan* control, +not a filter. One large repository can outweigh every shared cache combined, and a chart +containing it is a chart of one repository — so the ranking says, in the panel, that they were +left out. Turning the chip on starts a new deep scan that walks them (and turning it off starts +one that does not); it is disabled while a scan is running. `ak system --deep` scans without +project trees. + +### Two reclaimable tiers, never one total + +Reclaimable rows come in two tiers, rendered as two separate blocks because they are two +different promises: + +- **regenerable** — the owning tool refetches it on demand (package caches, superseded knowledge- + base copies, stale npx envs). This block states a total. +- **review** — plausible, but not safe to call removable: aged transcripts are the only copy of + the sessions they record, an extra runtime version may be the one a live toolchain resolves + through, a browser build may still be pinned. This block deliberately has **no total**. Its + bytes appear per row as context, and some rows show what is installed at that path rather than + a measured removable subset, labeled as such. + +The two are never added together. A combined "you could free N" would be the one number you would +act on and the one number the measurement cannot stand behind. Where a tier's own rows overlap on +disk, its total reads unknown-with-reason instead of counting the same bytes twice. Nothing here +removes anything; where a CLI already owns the cleanup, the row names it. + ### Reading the numbers honestly - **A section that has never been scanned says so.** It reads "not measured yet — press Rescan", @@ -209,7 +251,20 @@ this area. - **A failed measurement names its reason.** An unreadable directory degrades that node alone; its siblings and the rest of the scan are unaffected. - **Lines of code are approximate and say so**, counted by extension with `node_modules`, vendored - trees, and binary files excluded. + trees, and binary files excluded. Only *languages* carry lines. Frameworks, SDKs and tools are + shown as present or not — React does not own lines, the `.tsx` files do — and what the registry + could not name is listed by name rather than swept into an "Other" slice. +- **Projects are counted twice, and the two numbers differ.** The KPI reads + `N ever · M on disk`: *ever* is every project any host has ever recorded a session in, including + ones you have since deleted or moved; *on disk* is the subset that still exists, and only those + become rows in the Projects table — a deleted project has no bytes and no lines to measure. + A large gap is a fact about your history, not an error. +- **A project whose path cannot be recovered is counted, not invented.** Claude stores transcripts + in a directory name that encodes the project path lossily (`/`, `.` and `-` all become `-`), so + it cannot simply be decoded back. The path is read from the session record instead; where no + session recorded one and the encoded name cannot be confirmed against your filesystem, the + project is reported as unresolved and the *ever seen* count is shown as a floor. You will never + see a guessed path here. - **Growth per day is approximate too** — a file counts its whole size on the day it was last written, which is exact for append-only transcripts and over-counts rewritten databases. - **Some things cannot be attributed, and say that instead of guessing.** Codex transcripts are @@ -242,4 +297,15 @@ API requests. The dashboard remains localhost-only and offline-first. Usage, Obs System may show sensitive local project, transcript, or filesystem-path information; use them only where that local information may be viewed. System deliberately shows absolute paths — a storage breakdown that hides where the bytes live answers nothing — behind the same token-gated loopback -delivery as every other route. It reads file *metadata* only, never file contents. +delivery as every other route. + +What System reads is a short, fixed list: directory entries and file `stat` results; your +`.git/config` origin remote (so a project can link to its repository page — the kit never fetches +it, you click it); a linked worktree's `gitdir` pointer; the `cwd` **field** recorded at the top +of a session transcript, and OpenCode's per-session `directory` column, so a session can be +attributed to the right project; your projects' manifest **dependency names**, which are neither +evaluated nor resolved; and your source files' bytes, streamed through a fixed buffer purely to +count newlines. Each of those yields a path, a name, or a number. **No message, prompt, tool call, +tool result, or model output is ever read** — those stay in Usage and Observability, which have +their own contracts for them. The full enumeration is +[Machine footprint § The read surface](ddd/machine-footprint.md#the-read-surface). diff --git a/docs/adr/0025-machine-footprint-metrics.md b/docs/adr/0025-machine-footprint-metrics.md index 58bfaac..819a3e8 100644 --- a/docs/adr/0025-machine-footprint-metrics.md +++ b/docs/adr/0025-machine-footprint-metrics.md @@ -3,6 +3,10 @@ - **Status:** Implemented - **Date:** 2026-08-06 - **Updated:** 2026-08-06 — accepted and implemented; the open points below are resolved decisions +- **Updated:** 2026-08-07 — §7 replaced by an enumerated read surface (the collectors now read a + transcript head's `cwd` field and project manifests' dependency keys); §6 gains reclaimable + safety tiers; §8 and §9 added for the widened scan surface, the corrected brain/Playwright + figures, and project accounting - **Deciders:** agentic-kit maintainers - **Related:** [ADR-0005](0005-dashboard-in-page-routing-reveal.md), [ADR-0007](0007-maintainer-admin-local-telemetry.md), @@ -53,7 +57,10 @@ Everything needed is already locally readable at trust boundaries the kit alread - the catalog surfaces Integration management already projects into (`.claude/agents`, `.claude/commands`, skills listings, OpenCode's converted agents, MCP registrations); - the project catalog machine-wide discovery already assembles - ([ADR-0024](0024-project-intelligence-telemetry.md)'s transcript-cwd source). + ([ADR-0024](0024-project-intelligence-telemetry.md)'s transcript-cwd source); +- the third-party cache roots any developer machine accumulates — model weights, package caches, + toolchain installs — which are plain directories under `$HOME` and need naming, not access + (§8). The gap is a **domain and a UX**, not access. @@ -72,9 +79,11 @@ deliberately **not** part of: - **Historical usage** — no tokens, no cost, no model identity here; a byte of transcript on disk is a storage fact, not a spend fact; - **Observability** — no session lifecycle, no evidence confidence, no transcript content; this - domain reads `stat` metadata and manifest names, never message bodies; -- **Project intelligence** — no learning counters; the shared piece is only project *discovery*, - reused as a candidate-path source exactly as ADR-0024 reuses Observability's workspace store. + domain reads `stat` metadata, paths, and declared names, never message bodies (§7 enumerates + every read); +- **Project intelligence** — no learning counters; the shared piece is project *discovery* as a + candidate-path source, on the same boundary ADR-0024 draws when it reuses Observability's + workspace store, though this domain runs its own discovery (§9). ### 2. A fourth primary area: System @@ -94,19 +103,21 @@ left of Overview; the System area itself is unaffected): System secondary rail: [ Summary | Storage | Runtime | Catalog | Projects ] [ as of 2h ago · ⟳ rescan ] -Summary KPI band: install size · data size · live processes (combined RSS) · projects · - skills/agents/commands counts · machine free-space denominator. - "Largest consumers" strip (top-N across all categories). Freshness label. +Summary KPI band: install size · data size · live processes (combined RSS) · projects + ("N ever · M on disk") · skills/agents/commands counts · machine free-space + denominator. "Largest consumers" strip (top-20 roots, ranked or grouped by + ecosystem, project trees opt-in). Freshness label. Storage Breakdown tree: category → host → project → session. Transcripts vs ledgers/logs vs learning stores vs kit caches. Trailing-30d growth sparkline per host. Top-N largest - sessions/files. Advisory reclaimable candidates. + sessions/files. Advisory reclaimable candidates in two safety tiers, never one total. Runtime Live process table (host, pid, CPU%, RSS, uptime, bound project) + combined totals. Daemon census (count, age vs TTL, budget state). Child/MCP server process count. Catalog Deduplicated skills / agents / commands / plugins / MCP servers, each with a per-host presence matrix (which hosts carry it). Projects Table: project (name links to its git remote's web page when one exists — derived - from .git/config, "local only" otherwise), lines of code (by language), working-tree - bytes, .git bytes, node_modules bytes, last activity. + from .git/config, "local only" otherwise), lines of code (by language), detected + frameworks/SDKs/tools (presence only), working-tree bytes, .git bytes, + node_modules bytes, last activity. Rows are the on-disk subset; ever-seen is a count. ``` The window/refresh control sits in the secondary-actions slot, the same pattern as Usage's @@ -149,16 +160,22 @@ Metrics marked ✚ are additions beyond the requesting examples; the taxonomy is | Storage | ak's own caches: usage-index.json, observability-workspaces.json, footprint snapshot itself ✚ | known paths | | Storage | Top-N largest sessions / files ✚ | walk | | Storage | Trailing-30d growth per host (from mtime + size) ✚ | walk metadata | -| Storage | Advisory reclaimable candidates (stale npx envs, old transcripts, orphaned worktrees) ✚ | walk + heuristics | +| Storage | Advisory reclaimable candidates (stale npx envs, aged transcripts, superseded cache snapshots, regenerable package caches, redundant browser revisions, extra runtime versions, orphaned worktrees), each with a `safety` tier ✚ | walk + heuristics | +| Storage | Ranked largest consumers across ~50 curated third-party cache roots, grouped by ecosystem, with containment/residual accounting ✚ | consumer registry + walk | | Catalog | Unique skills / agents / commands across hosts, per-host presence matrix | host catalog surfaces | | Catalog | Plugins and registered MCP servers ✚ | settings surfaces | | Catalog | Config surface: managed CLAUDE.md/AGENTS.md block count, settings file sizes ✚ | managed-blocks registry | -| Projects | Project count; per project: LOC by language, working-tree bytes, `.git` bytes ✚, `node_modules` bytes ✚, last activity | project catalog + walk | +| Projects | Projects **ever seen** across hosts and the **on-disk** subset ✚; per on-disk project: LOC by language, detected frameworks/SDKs/tools by presence ✚, the unrecognized extension/dependency tail ✚, working-tree bytes, `.git` bytes ✚, `node_modules` bytes ✚, last activity | own cross-host discovery + walk | | Projects | Git remote web link per project (origin URL → GitHub/GitLab/etc. page; "local only" when absent) ✚ | `.git/config` remote parse — the admin collector's `parseRepoSlug` shapes, reused | LOC counting is a zero-dependency extension-bucketed line count by the kit's own bounded walker (no `cloc`/`tokei` dependency), excluding `node_modules`, vendored trees, and binary extensions — -stated as approximate, which is what the question needs. +stated as approximate, which is what the question needs. **Lines belong to languages only.** +Frameworks, SDKs and tools are detected by presence and carry no line count in the payload at +all: React does not own lines, the `.tsx` files do, and putting both on one proportional bar +would count the same bytes twice. What the registry cannot name is published as an +**unrecognized tail** — extensions by name, dependency keys by name — which turns the usual +silent "Other" slice into a to-do list a release can close. ### 5. Delivery @@ -168,26 +185,157 @@ stated as approximate, which is what the question needs. other dashboard route. - `GET /api/system?refresh=deep` — starts or attaches to the single-flight deep scan. The dashboard server is deliberately GET-only; a refresh is a re-*measurement* of local state, not - a mutation of user data, so it stays within that contract. + a mutation of user data, so it stays within that contract. `&trees=1|0` sets whether that scan + walks project working trees; it is a **measurement** parameter, not a view filter, because + trees that were never walked cannot be un-hidden client-side. - `ak system [--deep] [--json]` — CLI parity sharing the same collector, following the usage-scorecard precedent of one collector behind both surfaces. -### 6. Read-only; reclaimables are advisory +### 6. Read-only; reclaimables are advisory, in two safety tiers v1 computes reclaimable-space *candidates* and renders them with their rationale; it deletes nothing. Cleanup remains CLI-owned where it already lives (`ak x daemon-gc`, npx cache tooling). A future `ak system clean` would be its own decision with its own safety contract. -### 7. A deliberate path-visibility exception +Advisory is not enough on its own, because "advisory" is a tone of voice and users act on +numbers. Safety is therefore a **field**. Every candidate carries `safety`: + +- **`regenerable`** — the owning tool refetches it on demand: the npm content cache, the Homebrew + download cache, the brain's superseded `kb.bak-*` copies, stale npx envs. +- **`review`** — plausible but not safe to *state* as removable: aged transcripts (a transcript is + the only copy of the session it records, and Historical usage is denominated in them), extra + runtime versions (mise holds eight Node entries on this machine and some are aliases a live + toolchain resolves through), a browser revision that may still be pinned by an installed + package. + +**The two tiers are totalled separately and never added.** `combined` is `null` by design: a +single "you could free N" that mixed a regenerable cache with a possibly-live runtime tree would +be the one number a reader would act on and the one number this domain cannot stand behind. Two +supporting rules keep the per-tier totals honest — only rows whose `bytesMeaning` is `candidate` +are summable (an `installed` figure is context on a review row, not a claim about removable +space), and a tier whose rows describe overlapping paths reports its total as +unknown-with-reason rather than counting the same bytes twice. The `review` block is rendered +without a leading total at all, so it cannot read as available space. + +### 7. A deliberate path-visibility exception, and an enumerated read surface Observability's `publicLivePayload` reduces absolute paths to leaf names because its payloads describe *sessions* and a path is incidental provenance. In this domain the paths **are the subject matter** — a storage breakdown that hides where the bytes live answers nothing. The System payload therefore carries absolute paths, protected by the same token-gated loopback -delivery as everything else. File *contents* are never read: `stat` metadata, directory names, -catalog manifest names, and one narrow config read — `.git/config`'s remote URL, so a project -can link to its hosted repository page. Rendering that link stays inside the zero-egress -contract: the kit never fetches it; navigation is the user clicking a link in their own browser. +delivery as everything else. + +The content boundary is stated as an **enumeration**, not as "file contents are never read" — +that phrasing was true of the first collectors and became false the moment project discovery +needed a real path and the Projects table needed a language breakdown. What is read, and what is +taken from it: + +| Read | What is taken | What is never taken | +|------|---------------|---------------------| +| Directory entries, `lstat` | name, kind, size, mtime, blocks | anything inside a file | +| `.git/config` | the origin remote URL | every other key | +| `.git/worktrees//gitdir` | one path, bounded to 4 KB | — | +| A transcript's head (≤256 KB, ≤40 parsed lines) | the session's `cwd` **field** | every message, prompt, tool call, tool result and model output | +| OpenCode's session store (read-only) | the `directory` column | every other column and every message row | +| A project's own manifests (≤3 deep, ≤64 files, ≤512 KB each) | dependency **keys** | values, scripts, anything executable — nothing is evaluated or resolved | +| A project's own source files | the count of `\n` bytes | the text: each 64 KB chunk is counted and overwritten | + +Each of those yields a path, a name, or an integer. **No message body, prompt, tool call, tool +result, model output, or manifest value enters this domain, in any tier, on any path.** + +Three of the rows are justifications rather than mere disclosures. The transcript `cwd` read is +the *only* honest way to know which project a session belonged to — the alternative is decoding +Claude's transcript-directory name, which is a lossy encoding that a naive decode gets wrong for +85% of directories on a real corpus (§9). It is also the same read +`native-transcript-discovery.mjs` already performs for Observability at the same trust boundary, +and it is *discovery*, which supplies a candidate path and never a measurement. The manifest read +is the same class of datum as `.git/config`'s remote: a bounded read of a declaration file, for +names. And the source-file read is what "count lines of code" always meant — §4 sanctioned an +extension-bucketed line count from the first draft, and a line count is a byte scan; what is new +here is saying so. + +Rendering the git-remote link stays inside the zero-egress contract: the kit never fetches it; +navigation is the user clicking a link in their own browser. + +[Machine footprint § The read surface](../ddd/machine-footprint.md#the-read-surface) holds the +normative list; a read not on it is a defect, and adding one amends both documents. + +### 8. The scan surface is wider than the kit's own trees, and three figures were wrong + +The first implementation answered "where are *the kit's* bytes" and presented it as "what is +eating this disk". Those are different questions, and the gap was not a rounding error. Measured +on the authoring machine, the top consumers were **entirely invisible** to the panel: +`~/.ollama` 141.31 GB, `~/.lmstudio` 48.78 GB, `~/.cache/huggingface` 35.92 GB, `~/.npm/_cacache` +22.26 GB, mise installs 16.30 GB, Docker 15.47 GB, rustup 14.62 GB, the pnpm store 13.40 GB, the +brain 13.18 GB. The panel reported the npx cache at 6.64 GB as the machine's number one; it is +number eleven. + +The Summary strip's ranking is therefore its own collector over a curated registry of ~50 +third-party cache roots grouped by ecosystem, not a client-side sort over whatever other sections +happened to walk. A ranking whose breadth is an accident of what other sections needed is a +ranking that reads as an answer and is not one. Because these roots nest — `~/.npm` ⊃ `_cacache` +and `_npx`; `~/.cache` ⊃ huggingface, ruvnet-brain, puppeteer, pnpm, uv; `~/.local/share` ⊃ mise, +pnpm, claude, opencode; mise installs ⊃ node ⊃ the npm global root the Install section already +walks — the accounting rules are structural rather than editorial: **containment** (bytes counted +once, at the outermost row; enclosed rows are breakdowns, excluded from the ranking and the group +totals), **residuals** (a synthesized "everything else" row so a breakdown always sums to its +parent), and **project trees opt-in** (one repository here is 175 GB and would flatten the chart, +so `includeProjectTrees` defaults off and the exclusion is stated in the payload, never silent). +Absent roots are reported as absent, not ranked at zero; Docker's sparse VM image is measured in +allocated blocks because its apparent size (~4 TB) is larger than the disk. + +Two individual figures were wrong at the source, and both changed a headline: + +- **The brain was measured at its KB, not at its install root.** `kbDir()` alone under-reported + it by 85%: 1.9 GB of active KB inside a 13.18 GB cache root. The remainder is chiefly five + dated `kb.bak-*` copies the installer leaves behind on every update — ~11 GB that nothing + reads, and the single largest reclaimable on this machine — plus ~234 MB of embedding models. + The root is now measured whole *and* broken into components with a remainder row, because a + user watching a figure move from 1.9 GB to 13.18 GB is owed the reason. The upward resolution + is conservative: both `…/ruvnet-brain/kb` segments must match, so a relocated + `RUVNET_BRAIN_KB` cannot bill a shared volume to the brain. +- **Playwright's macOS location was not probed.** `playwright install` writes to + `~/Library/Caches/ms-playwright` on macOS; only the XDG and Windows paths were checked, so a + mac holding 1.86 GB of browser builds reported a *measured zero* — honest about the XDG path + that genuinely does not exist, wrong about the question the row asks. All three locations are + now probed, realpath-collapsed (two can be real at once), and summed into one row. + +The consumer walk carries raised caps of its own (depth 24, 2,000,000 entries) because the +walker's defaults were sized for install trees: rustup exhausted the default entry cap at 9.4 of +its 14.62 GB, and the pnpm store, LM Studio and `~/.claude` all bottomed out at depth 16. A +ranking whose deepest trees are systematically floors does not merely under-report them, it +**mis-orders** the answer. The caps are raised, not removed — a root that exhausts even these +still reports `≥`. + +### 9. Projects are counted twice, on purpose, and never given a fabricated path + +The Projects section publishes **`everSeen`** (every distinct project any host ever recorded a +session in, deletions included — the deletions are the point) and **`onDisk`** (the subset that +still resolves to a directory, the only projects a byte or line measurement can be taken of). +Here: 50 ever seen, 25 on disk, 21 of those git repositories. Only on-disk projects become table +rows; the vanished ones survive in `everSeen` rather than as unmeasurable rows. + +Discovery is this domain's own (`project-sources.mjs`), not `discoverRuvfloProjects()`. That +function answers "which projects carry ruflo learning state" by requiring `.claude-flow/neural/` +and reading only the 150 newest transcripts per host; both narrowings are right there and wrong +here — on this machine they collapse ~50 projects to 5. Sightings are de-duplicated by +**resolved real path**, so one project touched by Claude, Codex and OpenCode is one project and a +symlinked route is not a second one; a deleted path that cannot be resolved falls back to +`path.resolve` so it is still counted. + +**Claude's transcript-directory encoding is lossy and is not safely decodable.** +`~/.claude/projects//` maps `/`, `.` and a literal `-` all to `-`, so +`-Users-me-ai-agentic-kit` reads equally as `/Users/me/ai/agentic/kit` and +`/Users/me/ai/agentic-kit`. Measured on this corpus, naive `-`→`/` substitution resolves **8 of +52** directories — wrong for 85% of them. The path therefore comes from the transcript's declared +`cwd` (2,538 of 2,585 Claude transcripts here, all 175 Codex rollouts, and OpenCode's per-session +`directory` column). The encoded name is consulted only as a fallback for a directory where no +transcript declares a `cwd`, and even then only through a decoder that walks candidate segments +against the real filesystem and returns a path only when the filesystem confirms it — which can +name just 18 of the 52 on its own. When neither route resolves, the group is reported as +**`unresolved`**, contributes no row, and makes `everSeen` an explicit lower bound +(`complete: false`). It is never given a guessed path. Every row carries `origins` naming which +route produced it, so a fallback-derived path can never be read as a declared one. ## Consequences @@ -197,6 +345,10 @@ contract: the kit never fetches it; navigation is the user clicking a link in th already has trust-boundary access to — no new privileges, no egress. - Sprawl becomes visible and actionable: duplicate native builds, giant sessions, stale caches, and per-project bloat all surface with their locations. +- The consumer ranking answers the question a user actually asks — "what is eating this disk" — + rather than the narrower one the kit could answer without leaving its own trees. On the + authoring machine that moved nine roots totalling >320 GB from invisible to ranked, and + demoted the previously-reported number one to eleventh (§8). - The four-family dashboard (health / spend / activity / footprint) completes a coherent mental model, each family in its own context with clean boundaries. @@ -205,7 +357,11 @@ contract: the kit never fetches it; navigation is the user clicking a link in th - A fourth primary area is a permanent navigation-surface expansion and an explicit amendment of ADR-0005's three-area layout. - Deep scans on large corpora (multi-GB transcript trees, many projects) take real time and I/O; - the tiered model contains but does not eliminate that cost. + the tiered model contains but does not eliminate that cost, and widening the scan surface to + ~50 third-party cache roots (§8) widened that cost with it — tens of seconds on this machine. +- A curated registry of third-party cache locations is a maintenance surface: a tool that moves + its cache goes unmeasured until the registry learns the new path. It fails visibly (the row + reads absent with the path it looked at) rather than silently, which is the trade taken. - A persisted snapshot is a new on-disk artifact with its own staleness to manage honestly. ### Risks and mitigations @@ -216,8 +372,10 @@ contract: the kit never fetches it; navigation is the user clicking a link in th | Stale snapshot presented as current | Every deep-tier figure carries `asOf`; the freshness label and rescan control are part of the Summary view's contract, not an afterthought | | LOC figures treated as precise | Labeled approximate, extension-bucketed; the DDD doc forbids presenting them as authoritative | | Walker follows a symlink cycle or escapes a root | Symlinks never followed; depth and entry caps; one bad subtree degrades to unknown for that node only | -| Scope creep into cleanup/mutation | v1 invariant: this context mutates nothing; reclaimables are advisory rows with rationale | -| Boundary erosion into Usage/Observability | DDD invariants forbid tokens/cost, session evidence, and content reads; cross-links replace duplication | +| Scope creep into cleanup/mutation | v1 invariant: this context mutates nothing; reclaimables are advisory rows with rationale and a safety tier, and the tiers are never summed into one actionable number | +| Boundary erosion into Usage/Observability | DDD invariants forbid tokens/cost and session evidence; the read surface is enumerated (§7) and admits paths, names and counts only, never message content; cross-links replace duplication | +| A ranking that reads as an answer and is not one | Breadth is a curated registry, not an accident of what other sections walked; containment/residual accounting so nested roots cannot double-count; raised walk caps because a floored deep tree mis-*orders* the list; excluded categories stated in the payload | +| A size figure measured at the wrong root | Both known cases (brain KB vs cache root, Playwright's macOS location) are recorded in §8 with what they cost; new roots carry a component breakdown with a remainder row so a corrected figure arrives with its explanation | ## Resolved decisions @@ -277,7 +435,11 @@ All complete: - [Managed tools](../MANAGED-TOOLS.md) - `src/lib/footprint/` — the collectors: `walk.mjs` (the bounded walker and the `Measurement` vocabulary), `install.mjs`, `storage.mjs`, `runtime.mjs`, `catalog.mjs`, `projects.mjs`, - `snapshot.mjs` (the persisted deep-tier snapshot), `index.mjs` (the two-tier collector) + `consumers.mjs` (the ranked largest-consumers view and its containment/residual accounting), + `project-sources.mjs` (cross-host project discovery: ever-seen vs on-disk), + `stack-registry.mjs` + `stack-detect.mjs` (languages carry lines; frameworks/SDKs/tools are + presence-only; the unrecognized tail), `snapshot.mjs` (the persisted deep-tier snapshot), + `index.mjs` (the two-tier collector) - `src/commands/system.mjs` (the CLI twin) - `src/lib/live/win-process-survey.ps1` (the Windows stand-in for `ps` + `lsof`) - `src/lib/live/process-sessions.mjs` (the runtime survey this reuses) diff --git a/docs/ddd/machine-footprint.md b/docs/ddd/machine-footprint.md index a5355b6..ada5132 100644 --- a/docs/ddd/machine-footprint.md +++ b/docs/ddd/machine-footprint.md @@ -31,10 +31,10 @@ Each neighboring context owns a different kind of fact about overlapping raw mat boundaries are what keep all four honest: - **[Historical usage](context-map.md)** owns *spend* facts — tokens, API-equivalent cost, model - identity — parsed from transcript **content**. Machine footprint reads transcript **metadata** - (`stat` size, mtime, path) and never opens a message body. A 400 MB session is a storage fact - here and a token fact there; the two figures answer different questions and neither substitutes - for the other. + identity — parsed from transcript **content**. Machine footprint reads transcript `stat` + metadata, plus one declared field — the session's `cwd` — and never a message body + ([the read surface](#the-read-surface)). A 400 MB session is a storage fact here and a token + fact there; the two figures answer different questions and neither substitutes for the other. - **[Observability](observability.md)** owns *activity* evidence — session lifecycle, actors, per-field confidence, a protected transcript plane. Machine footprint's runtime census reuses the same process survey as a *source*, but publishes resource rows (CPU%, RSS, uptime), not @@ -42,47 +42,114 @@ boundaries are what keep all four honest: graph. - **[Project intelligence](project-intelligence.md)** owns *learning* trends from `.claude-flow/` state. Machine footprint measures those same directories only as bytes on - disk. The shared piece is project *discovery* — the candidate-path catalog — reused exactly as - ADR-0024 reuses Observability's workspace store: a path is not evidence, and discovery supplies - nothing this domain renders as a measurement. + disk. The shared piece is the *idea* of project discovery, not the implementation: this domain + runs its own (`project-sources.mjs`) because Intelligence's catalog answers "which projects + carry ruflo learning state" and collapses ~50 projects to 5 here. Either way a path is not + evidence, and discovery supplies nothing this domain renders as a measurement — exactly the + boundary ADR-0024 draws when it reuses Observability's workspace store. - **[Integration management](integration-management.md)** owns what *should* be deployed (bindings, projections, ownership). Machine footprint reports what *is* on disk and how big it is; catalog counts here are observed inventory, never desired state. Because every source is the local filesystem and the current-user process table — the same trust boundary `ak status` and the runtime survey already cross — no anti-corruption adapter guards -this domain's reads. What *is* guarded is content: this domain's collectors are structurally -metadata-only (they read directory entries, `stat` results, and manifest *names*), so transcript -text, prompt content, and tool payloads cannot enter the model at all. +this domain's reads. What *is* guarded is content, and the guarantee is narrower and more precise +than "the collectors never open a file": see [The read surface](#the-read-surface) for the +enumerated list of what is opened, what is taken out of it, and why every one of those reads is +still a metadata read. + +## The read surface + +Invariant 1 is an enumeration, not a slogan. Every read this domain performs is listed here; a +read not on this list is a defect, and adding one is an amendment to this document and to +[ADR-0025 §7](../adr/0025-machine-footprint-metrics.md#7-a-deliberate-path-visibility-exception-and-an-enumerated-read-surface). + +| Read | Collector | What is taken | What is never taken | +|------|-----------|---------------|---------------------| +| Directory entries and `lstat` | `walk.mjs`, every collector | name, kind, size, mtime, block count | anything inside a file | +| `.git/config` | `projects.mjs` | the origin remote URL | every other config key | +| `.git/worktrees//gitdir` | `storage.mjs` | one filesystem path, bounded to 4 KB | — | +| A transcript's **head** | `project-sources.mjs` | the session's `cwd` **field** | every message, prompt, tool call, tool result and model output in the file | +| OpenCode's session store | `project-sources.mjs` | the `directory` column, read-only | every other column, and every message row | +| A project's own manifests | `stack-detect.mjs` | dependency **keys** (and, for `path:`/`workspace:` entries, enough of the value to reject them) | manifest values, scripts, and anything executable | +| A project's own source files | `stack-detect.mjs` | the count of `\n` bytes, and whether byte 0 of the first chunk region is NUL | the text — each 64 KB chunk is counted and immediately overwritten | + +Three of those rows are new since the first draft of this document, and they are the reason this +section exists rather than a one-line invariant. + +**A transcript's `cwd` is a path, not content.** Project discovery opens each Claude and Codex +transcript, reads at most its first 256 KB, JSON-parses at most its leading 40 non-blank lines, +and takes exactly one field: `record.cwd` (Claude) or `record.payload.cwd` on the `session_meta` / +`turn_context` records that open a rollout (Codex). The parsed records are discarded at the end of +the loop; nothing but the path string survives the function. This is the same read +`native-transcript-discovery.mjs` already performs for Observability at the same trust boundary, +and it is what makes discovery honest: the alternative is guessing the path from the directory +name, which [Project accounting](#project-accounting) shows is wrong four times out of five. + +The head bound is a correctness statement as much as a cost one. A session's `cwd` is declared in +its opening records or nowhere, so reading further would cost the whole corpus (~2,700 files here) +to learn nothing — and it would put the collector's read window over message bodies for no gain. + +**A manifest's dependency keys are names, not code.** Stack detection reads `package.json`, +`Cargo.toml`, `go.mod`, `pyproject.toml`, `pom.xml`, `build.gradle`, `mix.exs`, `Gemfile`, +`pubspec.yaml` and their siblings to at most 3 directories deep, at most 64 of them, at most +512 KB each, and extracts dependency **keys**. Nothing is evaluated, executed, resolved or +fetched: `mix.exs` and `build.gradle` are Elixir and Groovy *source*, and they are scanned with +regexes, never run. The depth bound is not cosmetic — a scan of this machine found 1,884 +`Cargo.toml` files, nearly all of them inside Cargo's registry cache, and a deeper search would +report a dependency's dependencies as the project's own. + +**Counting lines requires reading bytes, and the ADR always required counting lines.** LOC was +sanctioned from the first draft (`§4`: "a zero-dependency extension-bucketed line count by the +kit's own bounded walker"). What was left implicit is that a line count is a byte scan: the +counter streams the file through one fixed 64 KB buffer, increments on `0x0a`, bails out on a NUL +byte in the first chunk (binary), and never retains, concatenates, decodes or emits the text. The +figure that leaves the function is an integer. + +The boundary that still holds, stated positively: **no message body, prompt, tool call, tool +result, model output, or manifest value enters this domain, in any tier, on any path.** Every +figure the System area renders is either a `stat` result, a count, or a path. + +Two neighbouring boundaries are untouched by the above. The `cwd` read is *discovery* +(invariant 9) and supplies a candidate path, never a measurement. And nothing here reads a +transcript for what Historical usage reads it for — tokens, cost and model identity stay in that +context, parsed from the message bodies this one does not open. ## Model ```text -Sources (all local, all metadata-only) +Sources (all local; see "The read surface" for exactly what is read) install roots (managed-tools detection) process table (runtime survey + pcpu/rss) transcript roots (~/.claude, ~/.codex, opencode store) + consumer registry (~50 curated third-party cache roots — consumers.mjs) known files (ledgers, tee logs, caches, indexes) - project catalog (discovery reuse) host catalog surfaces (agents/skills/commands/MCP) + project sources (own cross-host discovery) host catalog surfaces (agents/skills/commands/MCP) | v Collectors (bounded walkers; two tiers — src/lib/footprint/) - walk.mjs the one bounded walker + the Measurement vocabulary every collector shares + walk.mjs the one bounded walker + the Measurement vocabulary every collector shares + project-sources.mjs cross-host project discovery (everSeen / onDisk) + stack-registry.mjs + stack-detect.mjs — languages (lines) vs frameworks/SDKs/tools (presence) cheap tier: runtime.mjs census + known-file stats + carry-forward of last deep scan (TTL 60s) - deep tier: install.mjs + storage.mjs + catalog.mjs + projects.mjs (explicit, - single-flight) - snapshot.mjs persists the four deep sections; index.mjs is the two-tier collector façade + deep tier: install.mjs + storage.mjs + catalog.mjs + projects.mjs + consumers.mjs + (explicit, single-flight) + snapshot.mjs persists the deep sections; index.mjs is the two-tier collector façade | v -FootprintSnapshot { asOf, completeness, install, runtime, storage, catalog, projects } +FootprintSnapshot { asOf, completeness, install, runtime, storage, catalog, projects, consumers } install: HostInstallation[] { tool, version, installMethod, root, bytes, nativeAddons[] } runtime: RuntimeCensus { processes[], daemons[], totals } (ephemeral, never persisted) storage: StorageBreakdown { nodes: category → host → project → session, growth, topN, - reclaimables[] } + reclaimables[], reclaimSummary: { tiers[], combined: null } } catalog: CatalogInventory { skills[], agents[], commands[], plugins[], mcpServers[], each with per-host presence } - projects: ProjectFootprint[] { path, label, remote?: {host, slug, webUrl}, loc: {language: - lines}, treeBytes, gitBytes, nodeModulesBytes, lastActivity } + projects: ProjectFootprint[] { path, label, remote?: {host, slug, webUrl}, stack: {languages, + stack, unrecognized}, treeBytes, gitBytes, nodeModulesBytes, + lastActivity } + + counts { everSeen, onDisk, gitRepos, unresolved } + consumers: ConsumerRanking { rows[] (root | breakdown | residual), top[], groups[], + totals, absent[], unmeasured[], includeProjectTrees } | v Delivery @@ -124,6 +191,34 @@ caches (npx envs, browser binaries) are install-adjacent nodes with their own ro into a tool's tree. The machine's free-space figure is the section's denominator, so "the install is X GB" always has a "of Y free" next to it. +Two of those roots were measured at the wrong place, and the corrections are recorded here +because both changed a headline figure rather than a detail. + +**The brain is measured whole, not at its KB.** `install.mjs`'s `brainRoot()` resolves the +installer's own `…/ruvnet-brain/kb` layout upward to the cache root and measures that. Measuring +`kbDir()` alone under-reported this machine's brain by 85%: 1.9 GB of active KB inside a 13.2 GB +cache root. The remainder is not one thing, so it is not one number — the row carries components +(`BRAIN_COMPONENTS`) that break the root into the active KB, the superseded `kb.bak-*` copies the +installer leaves behind on every update (5 of them, ~11 GB together on this machine), the +embedding models (~234 MB), and a remainder row so the parts add up to the whole. A user watching +a figure move from 1.9 GB to 13.2 GB is owed that breakdown, and the `kb.bak-*` copies are the +answer: nothing reads them, and they are the largest single reclaimable on this machine. + +The upward resolution is deliberately conservative. Both path segments must match, because +`RUVNET_BRAIN_KB` can relocate the KB anywhere: a KB at `/mnt/data/kb` would make the parent a +directory the brain does not own, and billing a shared volume to the brain is a worse error than +under-reporting it. An unrecognized layout stays measured at the KB dir. + +**Playwright has three platform locations and they are one cache.** Only the XDG and Windows +paths were probed, so on macOS — where `playwright install` writes to +`~/Library/Caches/ms-playwright` — a machine holding 1.86 GB of browser builds reported a +*measured zero*: honest about the XDG path that genuinely does not exist, wrong about the +question the row asks. All three are now probed and collapse into a single row. Two can be real +at once (a cache migrated between layouts leaves both; on macOS `~/.cache` is sometimes a symlink +into `~/Library/Caches`), so candidates are realpath-collapsed before summing and an aliased +target is measured once. When none exists, the platform-canonical path is the one named, so the +measured zero still says where it looked. + ### Runtime census A point-in-time table of live host processes — reusing the existing current-user, argv-minimized @@ -131,8 +226,9 @@ survey and extending its `ps` read with `pcpu`/`rss` — plus the daemon census against the 12h TTL, budget state) and a child/MCP-server process count. The census is **ephemeral**: computed per request, never persisted into the snapshot file, because a process table is a moment, not a fact worth retaining, and persisting it would create a stale-liveness -trap. `snapshot.mjs` enforces this structurally — it serializes only the four deep-tier keys, so a -census handed to it is dropped rather than written. +trap. `snapshot.mjs` enforces this structurally — it serializes only the deep-tier keys (`install`, +`storage`, `catalog`, `projects`, `consumers`), so a census handed to it is dropped rather than +written. On **Windows** the census is real, not unsupported. `src/lib/live/win-process-survey.ps1` — a plain text script invoked the way the POSIX path already invokes `ps` and `lsof`, with no npm dependency @@ -157,8 +253,9 @@ A tree of `StorageNode`s: category (transcripts / ledgers-and-logs / learning st caches) → host → project → session leaf, each with bytes and file count. Derived views over the same walk: trailing-30d growth per host (from mtime + size — no content reads), top-N largest sessions and files, and advisory `ReclaimableCandidate` rows (stale npx envs, transcripts beyond -a stated age, orphaned worktrees), each carrying its rationale and its path. Candidates are -information, not actions — this context has no delete verb. +a stated age, superseded cache snapshots, regenerable package caches, redundant browser +revisions, extra runtime versions, orphaned worktrees), each carrying its rationale and its path. +Candidates are information, not actions — this context has no delete verb. Two honest limits belong with the numbers. **Growth is approximate and says so**: a file contributes its whole size on its mtime day, which is exact for append-only transcripts and @@ -167,6 +264,101 @@ transcripts carry no project attribution**: rollout paths are dated, not project project name lives inside the file, which this domain may not open. Those nodes are marked `attribution: 'none'` and render as "unattributable" — never blank, never zero. +### Largest consumers + +The storage tree answers "where are *the kit's* bytes"; it does not answer "what is eating this +disk", and for a while the panel pretended it did. The Summary strip's ranking was assembled in +the browser from whatever the install and storage sections happened to carry, which made it wrong +in the one way a size ranking must never be wrong: it named the npx cache (6.64 GB) this +machine's largest consumer while `~/.npm/_cacache` — three and a half times larger — was not +scanned at all. The real order on this machine begins `~/.ollama` 141.31 GB, `~/.lmstudio` +48.78 GB, `~/.cache/huggingface` 35.92 GB, `~/.npm/_cacache` 22.26 GB, mise installs 16.30 GB, +Docker 15.47 GB, rustup 14.62 GB, the pnpm store 13.40 GB, the brain 13.18 GB. The npx cache is +eleventh. + +`consumers.mjs` therefore owns the ranking as a first-class view over a **curated registry** of +~50 third-party cache roots — Ollama, LM Studio, Hugging Face, the npm/pnpm/yarn/bun caches, +rustup and Cargo, Go's module cache, uv and pip, Maven and Gradle, Playwright and Puppeteer, +mise, Homebrew, Docker — grouped by *ecosystem*, because the actionable question is which +toolchain is costing the disk: four Node package caches at 5 GB each is a Node answer, not four +unrelated rows. Third-party cache conventions live in that module rather than in `paths.mjs`, +which owns the kit's and its hosts' own locations and should stay auditable as such. + +A size ranking is trivially made dishonest, so three accounting rules are structural: + +- **Containment.** Roots nest — `~/.npm` contains `_cacache` and `_npx`; `~/.cache` contains + `huggingface`, `ruvnet-brain`, `puppeteer`, `pnpm` and `uv`; `~/.local/share` contains `mise`, + `pnpm`, `claude` and `opencode`; mise's installs tree contains the Node install that contains + the npm global root the install section already walks. Bytes are counted **once**, at the + outermost row (`kind: 'root'`); every enclosed row is a `kind: 'breakdown'` that explains its + parent instead of competing with it. Containment is *derived* from resolved paths, not + hand-declared, so a registry edit cannot silently start double-counting. Only roots are ranked + and only roots are summed. Without this rule the list is self-similar and the total is fiction. +- **Residuals.** A parent with breakdowns also gets a synthesized `:other` row — parent + minus its direct children — so a breakdown always adds up. This is what makes the brain row + legible: 13.18 GB, of which 1.9 GB is the active KB and ~11 GB is superseded `kb.bak-*` copies. + Merging the two figures and reporting only the KB are both ways of being wrong about the same + 11 GB. An unknown input makes the residual unknown, never a difference taken against a + fabricated zero, and a negative residual reports itself rather than rendering as a plausible + small number. +- **Project trees are opt-in.** One repository on a working machine can outweigh every shared + cache combined (175 GB here), and a bar chart containing it is a bar chart of one repository. + `includeProjectTrees` defaults to false and the exclusion is **stated in the payload** + (`consumers.projectTrees.reason`), never silent — an omitted category the reader cannot see is + the same failure as an unknown rendered as zero. + +Absent roots are not consumers: a cache root that does not exist is reported in `absent` with its +path — "we looked, it is not here" — and kept out of both the ranking and the group totals rather +than ranked as a zero-byte consumer. Unreadable roots are reported in `unmeasured` with their +errno and make the affected group total `partial`; they are never a zero. + +Two roots need a basis other than apparent size, and say so. Docker Desktop's VM image is a +sparse file whose apparent size here is ~4 TB against ~15 GB actually written, so it is measured +in **allocated blocks** and carries both figures plus a `basis` string; an apparent-size reading +would put a number larger than the disk at the top of the ranking. And figures the install or +projects scan already measured for the same path are **adopted** rather than re-walked, which is +why a row can name its `measuredBy`. + +This is deep-tier work by construction — a 22 GB content-addressable cache is ~10⁵ files — and it +carries its own raised walk caps (`CONSUMER_WALK_LIMITS`: depth 24, 2,000,000 entries) because +the walker's defaults were sized for install trees and transcript roots. Measured: rustup +exhausted the default entry cap at 9.4 of its 14.62 GB, and pnpm's content store, LM Studio and +`~/.claude` all bottomed out on depth 16. A ranking whose deepest trees are systematically floors +does not merely under-report them, it **mis-orders** the answer. Raised, not removed: a root that +exhausts even these still reports `≥`. + +### Reclaimable safety tiers + +Safety is a field, not a tone of voice. Every `ReclaimableCandidate` carries `safety`: + +- **`regenerable`** — the owning tool refetches it on demand. The npm content cache, the Homebrew + download cache, the brain's superseded `kb.bak-*` copies, stale npx envs. +- **`review`** — plausible but **not** safe to state as removable. Aged transcripts (a transcript + is the only copy of the session it records, and Historical usage is denominated in them); extra + runtime versions (mise has eight Node entries on this machine and some are the aliases a live + toolchain resolves through); a browser revision that may still be pinned by an installed + package. A review row is a pointer at something to look at. + +Each row also carries `bytesMeaning`: `candidate` (the bytes the row is actually about) or +`installed` (what is on disk at that path, offered as context on a review row that has no +defensible candidate subset). `keeps` names what was excluded from the figure because it is in +use, and `cleanupHint` names the CLI that already owns removal — documentation, not a command +this module runs. + +**The tiers never sum into one headline number.** `summarizeReclaimables` totals each tier +separately and sets `combined: null`, permanently. A combined "you could free N" that mixed a +regenerable cache with a runtime tree that may be live would produce the one number a reader +would act on and the one number this domain cannot stand behind. Two further rules keep even the +per-tier totals honest: only `bytesMeaning: 'candidate'` rows are summable, and a tier whose own +rows describe overlapping paths (an aged transcript can also sit under a project that no longer +exists) reports its total as unknown-with-reason rather than counting the same bytes twice — the +row count still stands, and every row still carries its own measured figure. + +The surfaces honour the split structurally: the `review` tier is rendered without a leading total +at all, and its bytes ride each row as context, so the block cannot be read as "N GB available +here". A row from a snapshot predating the `safety` field lands in `review` — the tier that +promises less — never in the one that reads as free space. + ### Catalog inventory Deduplicated `CatalogItem`s across hosts — skills, agents, commands, plugins, MCP servers — @@ -176,14 +368,80 @@ surfaces Integration management already projects into; item file contents are no what naming requires. The config-surface row (managed CLAUDE.md/AGENTS.md block count, settings file sizes) lives here because it answers the same "what is deployed" question. +### Project accounting + +**Two numbers, not one.** `discoverProjectSources()` publishes `everSeen` and `onDisk`, and they +are different questions: + +- **`everSeen`** — every distinct project any host has ever recorded a session in, *including* + the ones since deleted or moved. The deletions are the point, so they are never dropped. +- **`onDisk`** — the subset that still resolves to a directory, i.e. the only projects a byte or + line measurement can be taken of at all. Only these become table rows; the vanished ones + survive in `everSeen`, not as unmeasurable rows. + +On this machine: **50 ever seen, 25 still on disk, 21 of those git repositories.** The gap is the +figure, not an error. + +This domain deliberately does **not** reuse `discoverRuvfloProjects()`. That function answers a +different question — "which projects carry ruflo learning state" — by requiring a +`.claude-flow/neural/` directory and by reading only the 150 most-recently-modified transcripts +per host. Both narrowings are correct there and wrong here: on this machine they collapse ~50 +projects to 5. The Intelligence panel keeps that meaning; System has its own source with its own +stated method. + +**De-duplication is by resolved real path.** One project touched by Claude, Codex and OpenCode is +one project; a project reached through a symlink is the same project as the one reached directly. +Each sighting's `cwd` is `realpath`-resolved before it keys the map, falling back to +`path.resolve` when the target cannot be resolved — a deleted project has no real path and must +still be counted. A row therefore carries the set of hosts that saw it, its session count across +all of them, and its most recent sighting. + +**Claude's transcript-directory encoding is lossy, and this domain refuses to fake a decode.** +`~/.claude/projects//` encodes the project path by replacing `/`, `.` **and a literal +`-`** all with `-`, so `-Users-me-ai-agentic-kit` reads equally as `/Users/me/ai/agentic/kit` and +`/Users/me/ai/agentic-kit`. There is no safe pure-string decode. Measured on this machine's +corpus: naive `-`→`/` substitution resolves **8 of 52** directories — wrong for 85% of them. +`decodeClaudeProjectDir` therefore walks the candidate segments against the real filesystem and +returns a path only when the filesystem confirms it, under its own `lstat` budget; even that +verified walk can name only 18 of the 52 from the directory name alone. + +Which is why the encoded name is a **fallback, not the source**. The path comes from the +transcript's declared `cwd` (2,538 of 2,585 Claude transcripts here carry one; all 175 Codex +rollouts do; OpenCode's store carries an absolute `directory` per session). The decoder is +consulted only for a project directory where *no* transcript declares a `cwd` — one such group on +this machine. When neither route resolves, the group is counted as **`unresolved`** and +contributes no row: it is never given a fabricated path, and its existence makes `everSeen` a +**lower bound** rather than a total (`complete: false`). Every project row carries `origins` +saying which route named it (`cwd` or `encoded-dir`), so a fallback-derived path can never be +mistaken for a declared one. + +Codex is left out of the fallback entirely: its rollout directories are dated, not +project-scoped, so there is nothing to decode. + ### Project footprint -One `ProjectFootprint` per project in the shared discovery catalog: approximate lines of code -bucketed by language (the kit's own extension-bucketed walker — explicitly approximate, excluded -trees stated: `node_modules`, vendored, binary extensions), working-tree bytes, `.git` bytes, -`node_modules` bytes (kept separate precisely because it dominates and distorts), and last -activity. LOC figures are labeled approximate wherever they render; this domain forbids -presenting them as authoritative. +One `ProjectFootprint` per **on-disk** project: working-tree bytes, `.git` bytes, `node_modules` +bytes (kept separate precisely because it dominates and distorts), last activity, and a detected +stack. LOC figures are labeled approximate wherever they render; this domain forbids presenting +them as authoritative. + +**Lines belong to languages; frameworks are presence only.** `stack-detect.mjs` returns +`languages` with a line count, because a file extension is what a line belongs to — and `stack` +entries (frameworks, SDKs, tools) with **no `lines` field at all**. React does not own lines, the +`.tsx` files do, and stacking both on one proportional bar would count the same bytes twice. +Nothing downstream can make that mistake by accident because the number simply is not in the +payload. `stack-registry.mjs` is the versioned data behind both, and every detection carries its +`registryVersion` and its `via` (which manifest or signature named it). + +**The unrecognized tail is the point.** An extension the registry does not map is never counted +as lines — most unmapped extensions on a real machine are binaries and data — so it is tallied +*by name* instead, as is every declared dependency that matched no registry entry. That converts +the usual silent "Other" slice into a to-do list a release can close. The tail admits only what +belongs in it: an extension the registry has already ruled out as non-source (`.png`, `.sqlite`) +is a stated exclusion counted separately, and a key that is not shaped like an extension at all +(`.2026-08-06`, from a rotated log) collapses into a named bucket rather than minting a thousand +single-file "extensions". `STACK_EXCLUSIONS` ships attached to the figure, so no surface can +render a line count without being able to state what it left out. A project additionally carries an optional `remote` — host, slug, and derived web URL — parsed from `.git/config`'s origin remote through the same URL-shape handling the admin collector's @@ -216,8 +474,9 @@ running, and stops when it finishes. One deliberate divergence from Observability's delivery: absolute paths are **part of this payload**. `publicLivePayload`'s leaf-only rule exists to keep incidental provenance out of session payloads; here the path is the answer ("where are the bytes"), and the same token-gated -loopback delivery protects it. Transcript/message content remains structurally absent — the -collectors never read it, so delivery cannot leak it. +loopback delivery protects it. Message content remains structurally absent from the payload: the +collectors never take it out of a file ([the read surface](#the-read-surface)), so delivery has +nothing to leak. The CLI twin (`ak system`) renders the same collector output, `--json` emitting the collector's payload verbatim, following the one-collector-two-surfaces precedent of the usage scorecard. @@ -225,12 +484,16 @@ payload verbatim, following the one-collector-two-surfaces precedent of the usag ## Invariants -1. **Metadata only, ever.** Collectors read directory entries, `stat` results, manifest names, - `.git/config`'s remote URL, and — for the orphaned-worktree candidate, which no `stat` can - identify — `.git/worktrees//gitdir`, bounded to 4 KB and validated as an absolute path. - Both git reads are pointer metadata of the same class, enumerated here rather than left - implicit. No transcript, prompt, message, or tool-payload content enters this domain, in any - tier, on any path. +1. **Metadata only, ever — and the reads are enumerated, not asserted.** The complete list is + [The read surface](#the-read-surface); it is normative, and a read not on it is a defect. + Collectors read directory entries and `stat` results; `.git/config`'s remote URL; + `.git/worktrees//gitdir` (bounded to 4 KB, validated as an absolute path) for the + orphaned-worktree candidate, which no `stat` can identify; a transcript head's `cwd` **field** + and OpenCode's session `directory` column, for project discovery; a project manifest's + dependency **keys**; and a source file's bytes streamed through a fixed buffer to count + newlines. Every one of those yields a path, a name, or an integer. **No message body, prompt, + tool call, tool result, model output, or manifest value enters this domain, in any tier, on + any path**, and nothing read is retained past the function that read it. 2. **Unknown is never zero.** An unmeasured or failed measurement renders as unknown with a reason; a measured zero renders as zero. A total built over an unknown or capped input is `partial` and renders as a lower bound. No fabricated figures. @@ -239,7 +502,8 @@ payload verbatim, following the one-collector-two-surfaces precedent of the usag asks for one — opening the area never triggers it — and staleness past the stated threshold is surfaced as a nudge, not silently repaired. 4. **This context mutates nothing.** No delete, prune, or cleanup verb exists here; reclaimable - candidates are advisory rows with rationale. (The snapshot file it owns is the sole write.) + candidates are advisory rows with rationale, each carrying a `safety` tier and a + `bytesMeaning`. (The snapshot file it owns is the sole write.) 5. **The runtime census is ephemeral.** It is computed per request and never persisted; a stale process table is never replayed as liveness. 6. **Bounded walkers.** Symlinks are never followed; depth and entry caps apply; one unreadable @@ -249,17 +513,30 @@ payload verbatim, following the one-collector-two-surfaces precedent of the usag 8. **No spend, no activity, no learning facts.** Tokens/cost stay in Historical usage; session lifecycle/evidence stays in Observability; learning counters stay in Project intelligence. Cross-links, not duplication. -9. **Discovery supplies paths, not measurements.** The shared project catalog contributes - candidate locations only; everything rendered is measured by this domain's own collectors. +9. **Discovery supplies paths, not measurements.** The project catalog contributes candidate + locations only; everything rendered is measured by this domain's own collectors. A path that + cannot be resolved is reported as `unresolved` and never fabricated, which makes `everSeen` a + lower bound rather than a guess. 10. **Catalog counts are observed inventory.** They state what is on disk per host surface, never desired state, and never upgrade Integration management's ownership facts. 11. **LOC is approximate and says so.** Extension-bucketed line counts with stated exclusions; - no rendering presents them as authoritative. + no rendering presents them as authoritative. Lines belong to **languages** only: frameworks, + SDKs and tools are detected by presence and carry no line count in the payload at all, so no + surface can double-count the same bytes under a framework's name. 12. **Same delivery protections as the rest of the dashboard.** Loopback, token auth, GET-only, zero egress; the absolute-path exception is deliberate, documented, and content-free. 13. **Every platform reports what it can, and names what it cannot.** No section is switched off for a platform. Where a per-platform probe fails, that field alone degrades with its reason and the row keeps every other measurement; a row is never dropped for being unattributable. +14. **Sizes are counted once, and every exclusion is stated.** In the largest-consumers ranking + nested roots are counted at the outermost row only; enclosed rows are breakdowns that explain + their parent rather than competing with it; a residual row makes every breakdown add up to + its parent; absent roots are listed as absent rather than ranked as zero-byte consumers; and + a category excluded by default — project working trees — states its exclusion in the payload. +15. **Reclaimable tiers are never summed together.** `regenerable` and `review` are separate + promises with separate totals; `combined` is `null` by design. Only `bytesMeaning: + 'candidate'` rows are summable, and a tier whose rows describe overlapping paths reports + unknown-with-reason rather than counting the same bytes twice. ## Ubiquitous language additions @@ -269,13 +546,19 @@ normative and this table restates it for readers of this document. | Term | Meaning | |------|---------| | Footprint | The machine-resource cost of the toolchain: install bytes, runtime CPU/RSS, retained-data bytes, deployed inventory. The context's name; the surface is **System** | -| FootprintSnapshot | The persisted result of a deep scan: `asOf`, completeness, and the four deep-tier section models | +| FootprintSnapshot | The persisted result of a deep scan: `asOf`, completeness, and the deep-tier section models (install, storage, catalog, projects, consumers) | | Measurement | A value plus provenance: measured (with `asOf`), carried forward, or unknown-with-reason — unknown is never zero | | Partial measurement | A measured value known to be a lower bound because a contributing subtree was unreadable or capped; rendered as "≥ N" | | HostInstallation | One managed tool's install facts: version, install method, root, tree bytes, native addons | | RuntimeCensus | The ephemeral point-in-time table of live host processes, daemons, and machine denominators | | StorageNode | One node in the category → host → project → session breakdown: bytes + file count | | ReclaimableCandidate | An advisory row naming reclaimable space, its path, and its rationale — never an action | +| Safety tier | A candidate's `regenerable` (the owning tool refetches it) or `review` (plausible, not safe to call removable). The two are totalled separately and never combined | +| Bytes meaning | Whether a candidate's bytes are the `candidate` subset it is about, or the `installed` size at that path offered as context on a review row | +| Consumer root | A ranked top-level storage root. Nested rows are `breakdown`s of it, plus a synthesized residual, so bytes are counted once | +| Ever seen / on disk | `everSeen` is every project any host ever recorded a session in, deletions included; `onDisk` is the measurable subset. Different questions, never one number | +| Unresolved project | A transcript directory whose project path neither a declared `cwd` nor a filesystem-verified decode can name. Reported as such, never given a fabricated path; it makes `everSeen` a lower bound | +| Stack detection | Per-project `languages` (which carry lines) and `stack` — frameworks, SDKs, tools — which carry presence only, plus the unrecognized tail of extensions and dependency names the registry could not name | | CatalogItem | A deduplicated deployed artifact (skill, agent, command, plugin, MCP server) with a per-host presence matrix | | ProjectFootprint | One project's size facts: approximate LOC by language, tree/`.git`/`node_modules` bytes, last activity, and an optional git-remote web link ("local only" when absent) | | Deep scan | The explicit, user-triggered, single-flight full measurement pass that produces a FootprintSnapshot | @@ -288,5 +571,10 @@ normative and this table restates it for readers of this document. - [Historical usage / Observability / Project intelligence](context-map.md) — the neighboring contexts this domain is deliberately distinct from - [Dashboard guide](../DASHBOARD.md) -- `src/lib/footprint/` — the collectors; `src/commands/system.mjs` — the CLI twin; - `src/lib/live/win-process-survey.ps1` — the Windows process survey +- `src/lib/footprint/` — the collectors: `walk.mjs` (the bounded walker and the `Measurement` + vocabulary), `install.mjs`, `storage.mjs`, `runtime.mjs`, `catalog.mjs`, `projects.mjs`, + `consumers.mjs` (the ranked largest-consumers view), `project-sources.mjs` (cross-host project + discovery), `stack-registry.mjs` + `stack-detect.mjs` (languages, frameworks and the + unrecognized tail), `snapshot.mjs`, `index.mjs` +- `src/commands/system.mjs` — the CLI twin; `src/lib/live/win-process-survey.ps1` — the Windows + process survey diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index 625a0fe..1bd3db8 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -36,7 +36,8 @@ // ~60s) merged with the last persisted deep snapshot, // carried forward with ITS asOf. `?refresh=deep` starts // or attaches to the single-flight deep scan and returns -// immediately with progress state. +// immediately with progress state; `&trees=1|0` sets +// whether that scan walks project working trees. // // The status rows are gathered by SHELLING OUT to the installed CLI // (`node bin/agentic-kit.mjs status --json`) so we never duplicate status.mjs's @@ -1376,7 +1377,15 @@ export function startDashboard({ // it never rejects — the catch guards an injected collector that does // not honour that contract, so a bad one cannot take the process down // with an unhandled rejection. - Promise.resolve(collector.refreshDeep()).catch(() => {}); + // `trees` is a MEASUREMENT parameter, not a view filter: project + // working trees are only walked when it is set, and one large + // repository outweighs every shared cache combined — so the ranking + // has to be re-measured, not re-sorted. Absent means "keep whatever + // the collector already defaults to". + const trees = query.get('trees'); + Promise.resolve(collector.refreshDeep( + trees == null ? undefined : { includeProjectTrees: trees === '1' }, + )).catch(() => {}); // The payload predates the start by microseconds; re-stamp the live // scan block so this response reads "running", not "idle". if (typeof collector.scanState === 'function') payload.scan = collector.scanState(); diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index 79cbe9a..157e35f 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -1835,6 +1835,11 @@ export const JS = ` // carried forward with ITS timestamp. Nothing here ever renders an unmeasured // quantity as 0 — mhtml() has no code path that can. var SYSTEM=null, systemBusy=false, systemPollTimer=null; + // Consumers view state. consMode re-shapes rows the payload already carries; + // whether project trees were MEASURED is the server's fact, read back off the + // payload rather than mirrored here, so the chip can never claim a scope the + // numbers below it were not measured with. + var consMode="ranked"; // ── Measurement vocabulary (walk.mjs's, read-side) ── function mval(m){return m&&typeof m.value==="number"&&isFinite(m.value)?m.value:null;} @@ -1894,6 +1899,11 @@ export const JS = ` // ── odometer readout ── // Each digit rolls from 0 to its target. Under prefers-reduced-motion the // page-wide transition kill makes the same code paint the final value at once. + // Row height of one digit, in px. MUST equal .od's height in styles.mjs: the + // stack is scrolled by whole rows, so a mismatch shows two half digits. 30 + // because the digits are the Scorecard KPI's 27px — one type scale for both + // KPI bands, since they are one click apart. + var OD_H=30; function mountOdometers(root){ var els=(root||document).querySelectorAll(".od[data-od]"); for(var i=0;i0)?Math.max(0,Math.min(1,usedBytes/totalBytes)):0; - // A share small enough to be invisible as an arc is still a real answer, so - // the share is ALWAYS stated as a number. The arc is never padded up to a - // visible minimum: overdrawing a 0.2% slice to make it look like something - // would be a picture that disagrees with its own caption. - var share=(f*100)<0.1?"<0.1%":(f*100).toFixed(f<0.1?1:0)+"%"; - return '' - +'' - +(f>0.004?'' - +esc(fmtBytes(usedBytes)+" \\u00b7 toolchain")+"":"") - +''+esc(fmtBytes(usedBytes))+"" - +''+esc(share)+" of a "+esc(fmtBytes(totalBytes))+" disk" - +''+esc(fmtBytes(freeBytes))+" free" - +""; + // The disk denominator is a horizontal meter, not a radial: the question is + // "how much of this disk is the toolchain", one ratio, and a ratio does not + // need a card's worth of height. A segment thinner than 0.4% of the bar is not + // drawn — but its share is always STATED, because padding a sliver up to a + // visible width would draw a picture that disagrees with its own caption. + function diskSeg(bytes,total,color,label,opacity){ + var f=total>0?Math.max(0,Math.min(1,bytes/total)):0; + if(f<=0.004)return ""; + return ''; + } + function diskBand(installBytes,dataBytes,totalBytes,freeBytes){ + var used=installBytes+dataBytes; + var share=totalBytes>0?(used/totalBytes)*100:null; + var otherUsed=(freeBytes==null||totalBytes<=0)?null:Math.max(0,totalBytes-freeBytes-used); + var bar=diskSeg(installBytes,totalBytes,"var(--accent)", + "install \\u00b7 "+fmtBytes(installBytes)) + +diskSeg(dataBytes,totalBytes,"var(--accent)","retained data \\u00b7 "+fmtBytes(dataBytes),".5") + +(otherUsed==null?"":diskSeg(otherUsed,totalBytes,"var(--dim)", + "everything else on this disk \\u00b7 "+fmtBytes(otherUsed),".55")); + var facts=''+esc(fmtBytes(used))+" toolchain (" + +esc(fmtBytes(installBytes))+" install + "+esc(fmtBytes(dataBytes))+" retained)" + +(share==null?" \\u00b7 disk size unmeasured" + :" \\u00b7 "+esc(share<0.1?"<0.1%":share.toFixed(share<10?1:0)+"%") + +" of "+esc(fmtBytes(totalBytes))) + +" \\u00b7 "+(freeBytes==null?'free space unmeasured' + :""+esc(fmtBytes(freeBytes))+" free"); + // No separate legend row: the facts line already names install, retained and + // free, and each segment carries its own tooltip. A legend restating them + // would be a second line of height buying nothing. + return '
disk' + +'0?fmtBytes(totalBytes):"an unmeasured disk")+" used by the toolchain")+'">' + +bar+"" + +''+facts+"
"; } function svgDonut(slices,total,unit){ var C=2*Math.PI*56,off=0,arcs=""; @@ -2086,8 +2108,9 @@ export const JS = ` total+=v; counted++; } if(!counted)return unkHtml("no project reported a line count",false)+" lines"; - return "≈"+esc(fmtTok(total))+" lines across "+counted+" of "+list.length - +(projects.truncated?" \\u00b7 catalog truncated":""); + // One line, always: the caveat that the list was capped is accounting and + // belongs in the liner note, not in a caption that pushes five tiles taller. + return "≈"+esc(fmtTok(total))+" lines \\u00b7 "+counted+"/"+list.length+" counted"; } function rankedBars(rows){ if(!rows.length)return sysEmpty(NOT_SCANNED); @@ -2102,43 +2125,318 @@ export const JS = ` } // ── views ── + // The projects KPI states BOTH counts, because they answer different + // questions: everSeen is how many projects this machine has touched, onDisk is + // how many still exist to be measured. Rendering either alone under the word + // "projects" misreports the other. + // + // The older "count" field is NOT a stand-in for either. It is the number of + // rows the scan measured, and labelling it "ever seen" is how this panel + // reported 4 on a machine that has touched 50 — the wrong question answered + // under the right word. A snapshot that predates the cross-host census renders + // that field as exactly what it is, and the liner says why the others are gone. + function projectsValue(p){ + if(!p)return odCount(null); + if(!p.everSeen)return odCount(p.count)+'measured'; + var head=odCount(p.everSeen)+'ever'; + if(!p.onDisk)return head; + return head+'\\u00b7 '+mhtml(p.onDisk)+" on disk"; + } + // Liner note (data, never design): the two counts come from different + // populations and the de-duplication across hosts is the part a reader cannot + // infer from the number. + function projectsLiner(p){ + if(!p)return ""; + if(!p.everSeen){ + return "Projects \\u2014 this snapshot predates the cross-host project census, so " + +"it carries only the projects it measured, not how many this machine has ever " + +"touched. A rescan states both."; + } + var note='Projects \\u2014 ever: every ' + +"working directory any host recorded, de-duped across Claude, Codex and OpenCode by " + +"resolved real path, so one project used from two hosts counts once. on disk: the " + +"subset that still exists to measure."; + var g=mval(p.gitRepos); + if(g!=null)note+=" "+esc(fmtNum(g))+" are git repos."; + if(p.unresolved>0){ + note+=" "+esc(fmtNum(p.unresolved))+" path"+(p.unresolved===1?"":"s") + +" could not be decoded, so both counts are floors."; + } + if(p.truncated)note+=" The measured list is capped, so fewer rows than on-disk projects."; + return note; + } function renderSysSummary(d){ var install=d.install,storage=d.storage,catalog=d.catalog,projects=d.projects; - var rt=d.runtime||{},totals=rt.totals||{},machine=rt.machine||{}; + var rt=d.runtime||{},totals=rt.totals||{}; var kpis=document.getElementById("sys-kpis"); if(kpis){ + // Sub lines are kept to ONE line of facts each. A KPI band is read by + // scanning down the values; a tile whose caption wraps to a second line + // pushes every sibling tile taller for prose nobody scans. var counts=(catalog&&catalog.counts)||{}; kpis.innerHTML= kpiCard("install footprint",odBytes(install&&install.totals&&install.totals.installBytes), - mhtml(install&&install.totals&&install.totals.toolsPresent)+" managed tools \\u00b7 " - +mhtml(install&&install.totals&&install.totals.nativeAddons)+" native addons") + mhtml(install&&install.totals&&install.totals.toolsPresent)+" tools \\u00b7 " + +mhtml(install&&install.totals&&install.totals.nativeAddons)+" addons") +kpiCard("data retained",odBytes(storage&&storage.totals&&storage.totals.bytes), - "transcripts, ledgers, stores, caches") + "transcripts \\u00b7 ledgers \\u00b7 caches") +kpiCard("live processes",odCount(totals.processCount), - mhtml(totals.rssBytes,fmtBytes)+" resident \\u00b7 " - +mhtml(totals.cpuPercent,function(v){return v.toFixed(1)+"%";})+" CPU combined") - +kpiCard("projects",odCount(projects&&projects.count),locSummary(projects)) + mhtml(totals.rssBytes,fmtBytes)+" RSS \\u00b7 " + +mhtml(totals.cpuPercent,function(v){return v.toFixed(1)+"%";})+" CPU") + +kpiCard("projects",projectsValue(projects),locSummary(projects)) +kpiCard("catalog",odCount(counts.skill), - "skills \\u00b7 "+mhtml(counts.agent)+" agents \\u00b7 "+mhtml(counts.command)+" commands"); + "skills \\u00b7 "+mhtml(counts.agent)+" agents \\u00b7 "+mhtml(counts.command)+" cmds"); mountOdometers(kpis); } - var gauge=document.getElementById("sys-gauge"); - if(gauge){ + var kpiNote=document.getElementById("sys-kpis-note"); + if(kpiNote)kpiNote.innerHTML=projectsLiner(projects); + + var band=document.getElementById("sys-gauge"); + if(band){ var disk=(install&&install.disk)||null; var total=mval(disk&&disk.totalBytes),free=mval(disk&&disk.freeBytes); - var used=mval(install&&install.totals&&install.totals.installBytes),data=mval(storage&&storage.totals&&storage.totals.bytes); - if(total==null||used==null||data==null){ - gauge.innerHTML=sysEmpty(install?"the disk denominator needs install and storage figures; at least one is unmeasured.":NOT_SCANNED); + var used=mval(install&&install.totals&&install.totals.installBytes); + var data=mval(storage&&storage.totals&&storage.totals.bytes); + if(used==null||data==null){ + band.innerHTML=sysEmpty(install?"the disk band needs the install and storage totals; at least one is unmeasured.":NOT_SCANNED); }else{ - gauge.innerHTML=svgGauge(used+data,total,free==null?0:free, - "toolchain uses "+fmtBytes(used+data)+" of a "+fmtBytes(total)+" disk") - +'
install ' - +esc(fmtBytes(used))+" + retained "+esc(fmtBytes(data))+"" - +(free==null?'free space unmeasured':"")+"
"; + band.innerHTML=diskBand(used,data,total==null?0:total,free); } } - var cons=document.getElementById("sys-consumers"); - if(cons)cons.innerHTML=(!d.storage&&!d.install)?sysEmpty(NOT_SCANNED):rankedBars(topConsumers(d,6)); + renderSysConsumers(d); + } + + // ── largest consumers ── + // Every row the strip ranks comes from the consumers collector, which walks + // shared caches, model stores and toolchains the install and storage sections + // never see. When that section is absent the panel falls back to what those + // two sections DO carry and says so — a ranking whose scope is narrower than + // its title is exactly the misreading a liner note exists to prevent. + // The named parts of a row whose figure covers more than its label suggests. + // This is the whole liner note for the brain row: that row is the WHOLE + // ~/.cache/ruvnet-brain root, so it is many times the active-KB figure the + // install section reported on its own, and naming the parts says where the + // difference actually sits (superseded copies, embedding models) instead of + // leaving a number that looks like it grew overnight. + function consBreakdown(kids){ + if(!kids||!kids.length)return ""; + var list=kids.slice().sort(function(a,b){return (mval(b.bytes)||0)-(mval(a.bytes)||0);}); + var parts=[],unknowns=0,i; + for(i=0;iv*1.05)tail=fmtBytes(app)+" apparent"; + out.push({ + label:r.label,group:r.group,bytes:v,measure:r.bytes,tail:tail, + title:(r.path||r.pathPattern||"")+(r.accountingNote?" \\u2014 "+r.accountingNote:""), + // Only a row with breakdowns gets a visible note: those are the rows + // whose figure covers more than its name suggests, so the note is what + // makes the number readable rather than surprising. + note:kids[r.id]?(r.accountingNote||"")+consBreakdown(kids[r.id]):"", + color:r.group==="project-trees"?"var(--s2)":"var(--accent)" + }); + } + return out; + } + function consGroupRows(c){ + var gs=(c&&c.groups)||[],out=[],i; + for(i=0;imax)max=rows[i].bytes; + for(i=0;i0&&r.bytes!=null)?Math.max(1,(r.bytes/max)*100):0; + html+='
'+esc(r.label) + +(r.tail?''+esc(r.tail)+"":"")+"" + +'
'+(w>0?'':"")+"
" + +''+mhtml(r.measure,fmtBytes)+"
" + +(r.note?'
'+esc(r.note)+"
":""); + } + return html; + } + function consumerLiner(c,fallbackRows){ + if(!c){ + return "Ranked from the install and storage sections only \\u2014 this snapshot " + +"carries no consumers scan, so shared caches outside those two sections (npm's " + +"_cacache, model stores, language toolchains) are not counted here at all. " + +esc(fmtNum(fallbackRows))+" rows ranked."; + } + var ranked=mval(c.totals&&c.totals.rankedCount); + var a=c.accounting||{},t=c.projectTrees||{}; + // One line on the page, the collector's full accounting prose on hover: the + // reader needs the scope to read the ranking, not a paragraph to read the + // scope. + var full=[a.basis,a.containment,a.residuals,a.absent,t.included?null:t.reason] + .filter(Boolean).join(" "); + var lead=consMode==="ecosystem" + ? "Grouped by ecosystem, ranked roots only. " + : "Top "+esc(fmtNum(c.topN||20))+" of "+(ranked==null?"the":esc(fmtNum(ranked))) + +" measured roots. "; + return ''+lead + +"Counted: model stores, package and browser caches, language toolchains and the kit's " + +"own trees \\u2014 each at ONE level, so nesting never adds twice; absent roots are not " + +"ranked. " + +(t.included?"Project working trees are included in this measurement." + :"Project trees are NOT measured unless the chip is on \\u2014 one repo would flatten " + +"the ranking.") + +(c.unmeasured&&c.unmeasured.length + ? " "+esc(fmtNum(c.unmeasured.length))+" unmeasurable." + :"")+""; + } + function renderSysConsumers(d){ + var el=document.getElementById("sys-consumers"); + var note=document.getElementById("sys-consumers-note"); + var trees=document.getElementById("sys-cons-trees"); + var c=d.consumers||null; + if(trees){ + var on=!!(c&&c.includeProjectTrees); + trees.classList.toggle("on",on); + trees.setAttribute("aria-pressed",on?"true":"false"); + // Not a filter: whether trees were walked is decided at scan time, so the + // chip advertises the rescan it will start rather than pretending the + // answer is already on the client. + trees.title=on + ?"project working trees are in this ranking \\u2014 click to rescan without them" + :"project working trees are excluded \\u2014 click to rescan with them (a deep scan takes a while)"; + trees.disabled=!!(d.scan&&d.scan.running); + } + if(!el)return; + if(!c&&!d.storage&&!d.install){ + el.innerHTML=sysEmpty(NOT_SCANNED); + if(note)note.innerHTML=""; + return; + } + var rows=c?(consMode==="ecosystem"?consGroupRows(c):consRows(c)):topConsumers(d,20); + el.innerHTML=c?consumerRows(rows):rankedBars(rows); + if(note)note.innerHTML=consumerLiner(c,rows.length); + } + + // ── reclaimables (advisory, two tiers) ── + // ADR-0025 §6 and the storage collector's own header: this context has no + // delete verb, so nothing below is an action. The two safety tiers are + // rendered as SEPARATE blocks because they are separate promises — + // 'regenerable' is space the owning tool refetches by itself, 'review' is a + // pointer at something to look at. A review row therefore never gets the + // bytes-in-a-pill treatment: that pill is the visual grammar of "this much is + // yours to take back", and on a row that may be in use it would be a claim + // the measurement does not support. + var TIER_ORDER=["regenerable","review"]; + var TIER_FALLBACK_MEANING={ + regenerable:"The owning tool refetches this on demand.", + review:"Plausible but not safe to call removable \\u2014 review each one; this is not a " + +"total to sweep." + }; + function reclaimRow(r){ + var review=r.safety!=="regenerable"; + var meas=mhtml(r.bytes,fmtBytes); + var keeps=r.keeps||[],names=[],i; + for(i=0;i' + +(review?'review' + :''+meas+"") + +"
"+esc(r.label)+"
" + // On a review row the figure is CONTEXT and says which kind: 'installed' + // is what sits at that path, not a subset anyone measured as removable. + +(review?'
'+meas+" " + +esc(r.bytesMeaning==="installed"?"installed at this path":"across the matching files") + +" \\u2014 context, not a figure to reclaim
":"") + +'
'+esc(r.rationale||"")+"
" + +(names.length?'
in use and excluded from the figure: ' + +esc(names.join(", "))+"
":"") + +'
'+esc(r.path||"")+"
" + // Documentation, not an affordance: the CLI that already owns removal, + // spelled out so the reader runs it themselves somewhere else. + +(r.cleanupHint?'
removal lives in ' + +esc(r.cleanupHint)+"
":"") + +"
"; + } + function reclaimTier(safety,tier,rows){ + if(!rows.length)return ""; + var head=''+esc(safety)+""; + // The regenerable tier states its total; the review tier deliberately does + // not lead with one. Its bytes ride each row as context instead, so the + // block cannot be read as "N GB available here". + var figure=safety==="regenerable" + ? ""+(tier?mhtml(tier.bytes,fmtBytes):unkHtml("this snapshot carries no tier total",false)) + +" across "+esc(fmtNum(rows.length))+" row"+(rows.length===1?"":"s") + : esc(fmtNum(rows.length))+" row"+(rows.length===1?"":"s") + +' no tier total \\u2014 these are pointers, not a sum'; + var html='
'+head+figure+"
" + +'
' + +esc((tier&&tier.meaning)||TIER_FALLBACK_MEANING[safety]||"")+"
" + +'
'; + for(var i=0;i
"; + } + function renderSysReclaim(s){ + var rec=document.getElementById("sys-reclaim"); + var note=document.getElementById("sys-reclaim-note"); + if(note)note.innerHTML=""; + if(!rec)return; + if(!s){rec.innerHTML=sysEmpty(NOT_SCANNED);return;} + var list=s.reclaimables||[]; + if(!list.length){ + rec.innerHTML=sysEmpty("nothing crossed a reclaimable threshold \\u2014 a real, measured nothing."); + return; + } + var summary=s.reclaimSummary||null,tiers={},i; + for(i=0;i<((summary&&summary.tiers)||[]).length;i++)tiers[summary.tiers[i].safety]=summary.tiers[i]; + var html="",placed=0; + for(i=0;i'+esc(fmtNum(list.length-placed))+" row(s) carried no safety tier " + +"and are not shown.
":""); + if(note){ + note.innerHTML="Two tiers, never one total. " + +esc((summary&&summary.combinedNote) + ||"They are reported separately and never added: only the regenerable total is " + +"space a tool would rebuild by itself.") + +" Advisory rows only \\u2014 nothing here removes anything, by design."; + } } function renderSysStorage(d){ @@ -2219,25 +2517,7 @@ export const JS = ` +esc(String(g.basis||"file mtime and size only"))+""; } } - var rec=document.getElementById("sys-reclaim"); - if(rec){ - var list=(s&&s.reclaimables)||null; - if(!s){rec.innerHTML=sysEmpty(NOT_SCANNED);} - else if(!list||!list.length){rec.innerHTML=sysEmpty("nothing crossed a reclaimable threshold \\u2014 a real, measured nothing.");} - else{ - var advHtml=""; - for(i=0;i" - +'
'+esc(r.label)+"
" - +'
'+esc(r.rationale||"")+"
" - +'
'+esc(r.path||"")+"
" - +(r.cleanupHint?'
→ '+esc(r.cleanupHint)+"
":"") - +"
"; - } - rec.innerHTML=advHtml; - } - } + renderSysReclaim(s); var top=document.getElementById("sys-topsessions"); if(top){ var sess=(s&&s.topSessions)||null; @@ -2393,6 +2673,82 @@ export const JS = ` } } + // ── project stack ── + // Registry vocabulary, read-side. LANGUAGES carry lines and go on the bar; + // frameworks, SDKs and tools carry PRESENCE ONLY and go on chips beside the + // project — a chip has no width to misread as a quantity, which is exactly why + // they are chips and not another bar. + var STACK_KIND_LABEL={framework:"framework",sdk:"SDK",tool:"tool"}; + // Ranked steps of ONE hue, not six categories. styles.mjs caps the categorical + // series at four steps because past that they stop separating at 3:1; six + // languages of one project are an ORDER, so they render as one hue's ramp and + // are named in text underneath — identity from the label, size from the width. + var LANG_STEPS=[1,.8,.63,.49,.37,.27]; + var LANG_TOP=6; + /** Ranked languages, from the registry projection when present and from the + * older byLanguage map when the snapshot predates it. */ + function locLanguages(loc){ + if(!loc)return []; + if(loc.languages&&loc.languages.length){ + return loc.languages.map(function(r){ + return {name:r.name||r.id,lines:Number(r.lines)||0}; + }); + } + var out=[],k; + for(k in loc.byLanguage||{})out.push({name:k,lines:Number(loc.byLanguage[k])||0}); + return out.sort(function(a,b){return b.lines-a.lines;}); + } + function langCell(loc){ + var list=locLanguages(loc),total=mval(loc&&loc.total); + // An unmeasured line count is NOT an empty bar: an empty bar reads as a + // project with no code, which is a claim nobody made (invariant 2). + if(total==null)return unkHtml((loc&&loc.total&&loc.total.reason)||"lines were not counted",!loc); + if(!list.length||total<=0) + return '
'; + var bar="",names=[],shown=0,i; + for(i=0;i'; + names.push(list[i].name); + shown+=list[i].lines; + } + var rest=list.slice(LANG_TOP),restNames=[]; + for(i=0;ishown)bar+=''; + return '
'+bar+"
" + +'
' + +esc(names.join(" \\u00b7 "))+(rest.length?esc(" +"+fmtNum(rest.length)):"")+"
"; + } + /** Frameworks / SDKs / tools as chips. Capped, with the remainder named on + * hover rather than dropped. */ + function stackChips(stack){ + // A missing field means a snapshot older than the registry — not measured, + // which is a different statement from "this project uses nothing". + if(!stack)return '
stack not measured
'; + if(stack.status==="unknown") + return '
stack not measured
'; + var items=stack.items||[],chips="",i,rest=[]; + for(i=0;i'+esc(items[i].name)+""; + }else rest.push(items[i].name); + } + if(rest.length)chips+='+' + +esc(fmtNum(rest.length))+""; + // A measured project with no matches is a real "none", and it says so — + // rendering nothing would be indistinguishable from not having looked. + if(!chips)chips='none detected'; + return '
'+chips+"
"; + } function renderSysProjects(d){ var el=document.getElementById("sys-projects"); if(!el)return; @@ -2400,23 +2756,9 @@ export const JS = ` if(!p){el.innerHTML=sysEmpty(NOT_SCANNED);return;} var list=p.projects||[]; if(!list.length){el.innerHTML=sysEmpty("no project was discovered on this machine.");return;} - var LANG=["var(--s1)","var(--s2)","var(--s3)"],body="",i,j; + var body="",i; for(i=0;i0)locBar+=''; - shown+=langs[j].v; - } - if(locTotal>shown&&locTotal>0)locBar+=''; var tree=mval(pr.treeBytes),git=mval(pr.gitBytes),nm=mval(pr.nodeModulesBytes); var diskTotal=mval(pr.totalBytes),diskBar=""; if(diskTotal>0){ @@ -2438,28 +2780,101 @@ export const JS = ` name=esc(pr.label)+'
'+esc(rem&&rem.reason?rem.reason:"local only \\u2014 no git remote")+"
"; } var last=mval(pr.lastActivity); - body+=""+name+"" + body+=""+name+stackChips(pr.stack)+"" +''+mhtml(pr.loc&&pr.loc.total,function(v){return "~"+fmtTok(v);})+"" - +'
'+locBar+"
" + +""+langCell(pr.loc)+"" +''+mhtml(pr.totalBytes,fmtBytes)+"" +'
'+diskBar+"
" +''+(last==null?unkHtml((pr.lastActivity&&pr.lastActivity.reason)||"no readable entry",false) :esc(ago(Math.max(0,Math.round((Date.now()-last)/1000)))))+""; } - el.innerHTML='
lines: ' - +'1st language' - +'2nd' - +'3rd' - +'other' - +'disk: tree' + el.innerHTML='
' + +'lines: top '+LANG_TOP+' languages, darkest first' + +'' + +'' + +'' + +' the rest' + +'disk: tree' +'.git' - +'node_modules
' - +'
' + +'node_modules' + +'chips are frameworks, SDKs and tools \\u2014 presence only, never lines' + // The stack column needs a floor: chips that wrap one-per-line turn every + // project into a ten-line row, so the table scrolls horizontally in its own + // wrapper rather than squeezing the name column. + +'
Project
' + +'' +'' +'' +''+body+"
Project & stackLines ≈By languageDisktree · .git · node_modulesLast active
" - +'
'+mhtml(p.count)+" projects discovered \\u00b7 line counts are approximate: " - +"extension-bucketed, with node_modules and vendored trees excluded
"; + // The table can only hold projects that still exist — a deleted one has no + // bytes and no lines. Saying so next to the two counts is what keeps the + // shorter table from reading as a shrinking machine. + +'
' + +(p.everSeen + ?mhtml(p.everSeen)+" projects ever seen across all hosts, " + +(p.onDisk?mhtml(p.onDisk):"some")+" still on disk and listed here" + :mhtml(p.count)+" projects measured (this snapshot predates the ever-seen count)") + +". Line counts are approximate: extension-bucketed, with node_modules and vendored " + +"trees excluded. Frameworks, SDKs and tools are detected by PRESENCE and carry no " + +"line count of their own.
"; + renderSysUnrecognized(p); + } + + // "Other" as a to-do list, not a rounding bucket. Every extension the registry + // could not map and every declared dependency it could not name is listed BY + // NAME with how much of it there is — which is what makes excluding an + // unmapped extension from the line count an accounted decision rather than a + // silent loss. + function renderSysUnrecognized(p){ + var el=document.getElementById("sys-unrecognized"); + if(!el)return; + if(!p){el.innerHTML=sysEmpty(NOT_SCANNED);return;} + var u=p.unrecognized; + if(!u){ + el.innerHTML=sysEmpty("this snapshot predates the stack registry, so it carries no " + +"unrecognized tail."); + return; + } + // Ranked already; the head of each list is what a release would actually + // close, and the tail past the cap is named on hover rather than dropped. + var SHOW=24; + var ext=u.extensions||[],dep=u.dependencies||[],chips="",i,rest=[]; + for(i=0;i=SHOW){rest.push(ext[i].ext+" "+fmtNum(ext[i].files));continue;} + chips+=''+esc(ext[i].ext) + +""+esc(fmtNum(ext[i].files))+""; + } + if(rest.length)chips+='+' + +esc(fmtNum(rest.length))+""; + var deps="",depRest=[]; + for(i=0;i=SHOW){depRest.push(dep[i].name);continue;} + deps+='' + +esc(dep[i].name)+""; + } + if(depRest.length)deps+='+' + +esc(fmtNum(depRest.length))+""; + // Each total is stated, or its absence is: a capped list reports null rather + // than a smaller number presented as complete. + var counts=(u.extensionsTotal==null + ?"The extension list is capped, so the distinct count is a floor." + :esc(fmtNum(u.extensionsTotal))+" distinct unmapped extension(s).") + +" "+(u.dependenciesTotal==null + ?"The dependency list is capped too." + :esc(fmtNum(u.dependenciesTotal))+" unnamed dependenc" + +(u.dependenciesTotal===1?"y":"ies")+"."); + el.innerHTML=(chips?'
'+chips+"
":"") + +(deps?'
declared dependencies the registry does not name
' + +'
'+deps+"
":"") + +(!chips&&!deps?sysEmpty("the registry named everything it saw \\u2014 a measured nothing."):"") + +'
Counted over ' + +esc(fmtNum(u.projectsMeasured||0))+" measured project(s) by registry " + +esc(String(p.registryVersion||"?"))+". "+counts + +" Unmapped extensions are excluded from the line count \\u2014 which is exactly why " + +"they are listed by name here rather than folded into it.
"; } // Freshness is a contract, not a caption (ADR-0025 §3): every deep figure on @@ -2501,12 +2916,18 @@ export const JS = ` if(SYSTEM.error){ var ids=["sys-kpis","sys-gauge","sys-consumers","sys-donut","sys-hostsplit","sys-growth", "sys-reclaim","sys-topsessions","sys-procs","sys-mem","sys-daemons","sys-radar", - "sys-catcounts","sys-matrix","sys-projects"]; + "sys-catcounts","sys-matrix","sys-projects","sys-unrecognized"]; var msg=sysEmpty(SYSTEM.error+(SYSTEM.reason?" \\u2014 "+SYSTEM.reason:"")); for(var i=0;i SYSTEM

Summary

-

The four families in one glance: install size, retained data, live resource use, and - deployed inventory — every deep-tier figure stamped with when it was measured.

+

Install size, retained data, live resource use and deployed inventory — each + deep-tier figure stamped with when it was measured.

-
-

Disk denominator

+
+
-
-

Largest consumers — all categories

-
+
+
+

Largest consumers

+ +
+ + + +
+
+
+
@@ -517,12 +529,19 @@ ${LIVE_HTML}

Per-host split by category

-
+

Growth — bytes added per day

-
+ +

Reclaimable — advisory only

+
@@ -585,6 +604,13 @@ ${LIVE_HTML}

Project footprints

+ +
+

Unrecognized — what “other” is made of

+
+
diff --git a/src/lib/dashboard/styles.mjs b/src/lib/dashboard/styles.mjs index a7f29e5..1325f04 100644 --- a/src/lib/dashboard/styles.mjs +++ b/src/lib/dashboard/styles.mjs @@ -406,6 +406,7 @@ body.gated .band,body.gated .tabbar,body.gated main{display:none} } .chipf.on{border-color:var(--accent); color:var(--accent); background:var(--accent-soft)} .chipf:focus-visible{outline:2px solid var(--accent); outline-offset:1px} +.chipf:disabled{opacity:.5; cursor:not-allowed} .view[hidden]{display:none} /* hero KPIs */ @@ -873,11 +874,14 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} sy-prefixed for the same collision reason. A 12-column grid so each card can declare the width its chart form actually needs; charts are inline SVG built from the payload, never an image and never a remote asset. */ -.sy-grid{display:grid; grid-template-columns:repeat(12,1fr); gap:14px} +.sy-grid{display:grid; grid-template-columns:repeat(12,1fr); gap:10px} +/* Card chrome is deliberately tight. A System view is a dense band of facts — + padding and inter-row gap that would flatter one hero chart is, across five + stacked cards, most of a screen spent on framing rather than measurement. */ .sy-card{ - grid-column:span 12; display:flex; flex-direction:column; gap:10px; min-width:0; + grid-column:span 12; display:flex; flex-direction:column; gap:8px; min-width:0; background:var(--panel); border:1px solid var(--line); border-radius:var(--r); - padding:15px 17px; box-shadow:var(--shadow); + padding:13px 15px; box-shadow:var(--shadow); } .sy-3{grid-column:span 3}.sy-4{grid-column:span 4}.sy-5{grid-column:span 5} .sy-6{grid-column:span 6}.sy-7{grid-column:span 7}.sy-8{grid-column:span 8} @@ -896,18 +900,78 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} .sy-asof{font-family:var(--mono); font-size:11.5px} .sy-asof[data-stale="1"]{color:var(--warn)} .sy-scan{color:var(--accent)} -/* KPI band + odometer readout */ -.sy-kpis{grid-column:span 12; display:grid; gap:14px; grid-template-columns:repeat(auto-fit,minmax(178px,1fr))} -.sy-kpi{background:var(--panel); border:1px solid var(--line); border-radius:var(--r); padding:14px 16px; box-shadow:var(--shadow)} -.sy-kpi .lbl{font-size:10.5px; font-weight:650; letter-spacing:.09em; text-transform:uppercase; color:var(--ink-dim)} -.sy-kpi .val{display:flex; align-items:baseline; gap:5px; margin-top:6px; min-height:34px} -.sy-kpi .unit{font-size:13px; color:var(--ink-2)} -.sy-kpi .sub{font-size:11.5px; color:var(--ink-2); margin-top:4px} -.od{display:inline-flex; overflow:hidden; height:34px; font-family:var(--mono); font-variant-numeric:tabular-nums} -.od .dcol{display:inline-block; height:34px; overflow:hidden} +/* KPI band + odometer readout. Dimensions are Usage's .kpi rhythm EXACTLY — + 15px 16px of padding and a 27px value — not a second scale: the two bands are + one click apart, and a System tile that towers over a Scorecard tile reads as + a different, more important kind of fact than it is. OD_H in client.mjs must + equal the odometer height below — the digit stack is translated by whole rows, + so a mismatch shows two half digits. */ +.sy-kpis{grid-column:span 12; display:grid; gap:12px; grid-template-columns:repeat(auto-fit,minmax(168px,1fr))} +.sy-kpi{background:var(--panel); border:1px solid var(--line); border-radius:var(--r); padding:15px 16px; box-shadow:var(--shadow)} +.sy-kpi .lbl{font-size:10.5px; font-weight:600; letter-spacing:.09em; text-transform:uppercase; color:var(--ink-dim)} +.sy-kpi .val{display:flex; align-items:baseline; gap:5px; margin-top:5px; min-height:30px; flex-wrap:wrap} +.sy-kpi .unit{font-size:12px; color:var(--ink-2)} +.sy-kpi .sub{font-size:11.5px; color:var(--ink-2); margin-top:5px} +.od{display:inline-flex; overflow:hidden; height:30px; font-family:var(--mono); font-variant-numeric:tabular-nums} +.od .dcol{display:inline-block; height:30px; overflow:hidden} .od .dstack{display:flex; flex-direction:column; transition:transform 1.1s cubic-bezier(.2,.7,.2,1)} -.od .dstack span{height:34px; line-height:34px; font-size:27px; font-weight:700; text-align:center; min-width:.62em} -.od .lit{font-size:27px; font-weight:700; line-height:34px} +.od .dstack span{height:30px; line-height:30px; font-size:27px; font-weight:700; letter-spacing:-.028em; text-align:center; min-width:.62em} +.od .lit{font-size:27px; font-weight:700; line-height:30px; letter-spacing:-.028em} +/* Liner note: what a panel counted and how, in the muted voice the genuine + caveats already use. It qualifies the DATA — a figure whose accounting + changed cannot be read correctly without it, which is why it renders next to + the number rather than in a doc. */ +.sy-liner{font-size:11.5px; color:var(--ink-dim); line-height:1.45} +.sy-liner b{color:var(--ink-2); font-weight:600} +.sy-grid > .sy-liner{grid-column:span 12; margin-top:-5px} +/* Disk denominator as a horizontal meter. The radial spent a full card's height + stating one ratio; the band states the same ratio plus its parts in one line. + It also drops the card chrome entirely — a panel, a border and a shadow around + a single line of text is 20px of container spent framing 18px of content, and + the Summary is meant to read as a dense band of facts rather than a stack of + tall cards. It keeps its own hairline rules so it still reads as a strip. */ +.sy-band{ + background:none; border:0; border-radius:0; box-shadow:none; + border-top:1px solid var(--line); border-bottom:1px solid var(--line); + padding:5px 2px; gap:0; +} +.sy-diskband{display:flex; align-items:center; gap:13px; flex-wrap:wrap} +.sy-diskband .dk-lbl{ + flex:none; font-size:10.5px; font-weight:600; letter-spacing:.09em; + text-transform:uppercase; color:var(--ink-dim); +} +.sy-diskband .dk-meter{ + flex:1 1 200px; min-width:140px; height:12px; border-radius:100px; + background:var(--panel-2); overflow:hidden; display:flex; +} +.sy-diskband .dk-meter i{height:100%; min-width:2px} +/* Wraps to its own line rather than forcing the band wider than the viewport: + a facts string that cannot shrink is how a panel makes the whole page scroll + sideways on a phone. */ +.sy-diskband .dk-facts{flex:1 1 auto; min-width:0; font-size:12px; color:var(--ink-2)} +.sy-diskband .dk-facts b{color:var(--ink); font-family:var(--mono); font-weight:700} +/* Consumers: twenty ranked rows in a fixed-height scroller, about five visible. + The scroll belongs to this container — the page body must never scroll + sideways or grow a screen-tall list to state a ranking. */ +.sy-ctl{display:flex; gap:6px; flex-wrap:wrap} +.sy-scroll{ + max-height:110px; overflow-y:auto; overflow-x:hidden; + overscroll-behavior:contain; padding-right:4px; +} +.sy-crow{ + display:grid; grid-template-columns:minmax(110px,200px) 1fr 96px; gap:10px; + align-items:center; font-size:12.5px; padding:3px 0; +} +.sy-crow .n{color:var(--ink-2); overflow:hidden; text-overflow:ellipsis; white-space:nowrap} +.sy-crow .g{color:var(--ink-dim); font-size:10.5px; margin-left:6px} +.sy-crow .sy-track{height:11px} +.sy-crow .v{text-align:right; font-family:var(--mono); font-size:11.5px; color:var(--ink)} +.sy-cnote{font-size:11px; color:var(--ink-dim); line-height:1.4; margin:0 0 5px 96px} +@media(max-width:600px){ + .sy-crow{grid-template-columns:1fr 84px} + .sy-crow .n{grid-column:1/-1} + .sy-cnote{margin-left:0} +} /* Horizontal magnitude bars (ranked + stacked) */ .sy-bars{display:flex; flex-direction:column; gap:8px} .sy-bar{display:grid; grid-template-columns:minmax(96px,150px) 1fr 78px; gap:10px; align-items:center; font-size:12.5px} @@ -930,6 +994,31 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} .sy-table a{color:var(--accent); text-decoration:none; font-weight:600} .sy-table a:hover,.sy-table a:focus-visible{text-decoration:underline} .sy-sub{font-family:var(--mono); font-size:10.5px; color:var(--ink-dim); margin-top:2px} +/* Stack chips: frameworks, SDKs and tools are PRESENCE facts, so they get a + shape with no length to misread as a quantity. The kinds differ by weight of + outline, not by hue — three more hues here would compete with the four-step + data series next to them for no gain in meaning. */ +.sy-chips{display:flex; flex-wrap:wrap; gap:4px; margin-top:5px} +.sy-chip{ + font-size:10.5px; line-height:1.5; border-radius:100px; padding:1px 7px; + border:1px solid var(--line-2); color:var(--ink-2); background:var(--panel-2); + white-space:nowrap; cursor:help; +} +/* A weight ladder, not three hues: framework reads strongest, tool faintest. + Accent is this page's ACTION colour — six accent chips per project row would + read as six buttons, and they would out-shout the data bars beside them. */ +.sy-chip[data-kind="framework"]{border-color:var(--line-2); color:var(--ink); background:var(--panel-2)} +.sy-chip[data-kind="sdk"]{border-color:var(--line-2); color:var(--ink-2); background:transparent} +.sy-chip[data-kind="tool"]{border-style:dashed; background:transparent; color:var(--ink-dim)} +/* The unrecognized tail: a named to-do list, so each chip carries its count. */ +.sy-chip[data-kind="ext"]{font-family:var(--mono); background:transparent} +.sy-chip[data-kind="dep"]{font-family:var(--mono); background:transparent; border-style:dashed} +.sy-chip b{color:var(--ink-dim); font-weight:600; margin-left:5px} +.sy-chip.more{border-style:dashed; color:var(--ink-dim)} +.sy-subhead{font-size:10.5px; font-weight:600; letter-spacing:.06em; text-transform:uppercase; color:var(--ink-dim); margin-top:10px} +/* Language names under the stacked bar — identity in text, magnitude in the bar. */ +.sy-langs{font-size:10.5px; color:var(--ink-dim); margin-top:3px; max-width:210px; + overflow:hidden; text-overflow:ellipsis; white-space:nowrap; cursor:help} .sy-inbar{height:9px; border-radius:3px; background:var(--panel-2); min-width:80px; display:flex; gap:2px; overflow:hidden} .sy-rss{display:flex; gap:8px; align-items:center} .sy-dot{display:inline-block; width:8px; height:8px; border-radius:50%; margin-right:6px} @@ -940,12 +1029,40 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} .sy-tile{flex:1; min-width:92px; background:var(--panel-2); border-radius:var(--r-sm); padding:10px 12px} .sy-tile .t-v{font-family:var(--mono); font-size:20px; font-weight:700; color:var(--ink)} .sy-tile .t-l{font-size:11px; color:var(--ink-2); margin-top:2px} +/* Reclaimables — two safety tiers, deliberately NOT one list. + 'regenerable' is space the owning tool rebuilds by itself, so its rows lead + with the byte count in a pill. 'review' is a pointer at something to look at: + its rows lead with the WORD, and their bytes render as muted context text. + That difference is the whole point — the pill is the visual grammar of "this + much is yours to take back", and putting it on a row that may be in use would + claim something the measurement does not support. Neither tier has, or may + ever have, a delete affordance (ADR-0025 §6). */ +/* Rows flow into columns at width rather than stacking: a rationale paragraph + set across a full 1100px card is an unreadable measure, and nine of them + stacked is a screen of scrolling to state an advisory. */ +.sy-tier{margin-bottom:14px} +.sy-tier:last-child{margin-bottom:0} +.sy-tier-rows{display:grid; gap:0 22px; grid-template-columns:repeat(auto-fill,minmax(330px,1fr))} +.sy-tier-head{display:flex; align-items:baseline; gap:8px; flex-wrap:wrap; font-size:12.5px; color:var(--ink-2)} +.sy-tier-head b{color:var(--ink); font-family:var(--mono); font-weight:700} +.sy-tier-head .g{color:var(--ink-dim); font-size:11px} +.tier-pill{ + font-size:10px; font-weight:700; letter-spacing:.06em; text-transform:uppercase; + border-radius:100px; padding:2px 8px; +} +.tier-pill[data-safety="regenerable"]{color:var(--ok); background:var(--ok-soft)} +.tier-pill[data-safety="review"]{color:var(--ink-2); background:var(--panel-2); border:1px solid var(--line-2)} +.sy-tier-note{font-size:11.5px; color:var(--ink-dim); line-height:1.45; margin:3px 0 4px} .sy-adv{display:flex; align-items:flex-start; gap:10px; padding:8px 0; border-bottom:1px solid var(--line); font-size:12.5px} .sy-adv:last-child{border-bottom:0} -.sy-adv .tag{ - flex:none; font-size:10.5px; font-weight:700; border-radius:100px; padding:2px 9px; - color:var(--warn); background:color-mix(in srgb,var(--warn) 14%,transparent); -} +.sy-adv .tag{flex:none; font-size:10.5px; font-weight:700; border-radius:100px; padding:2px 9px} +.sy-adv .tag.regen{color:var(--ok); background:var(--ok-soft); font-family:var(--mono)} +.sy-adv .tag.review{ + color:var(--ink-2); background:transparent; border:1px dashed var(--line-2); + text-transform:uppercase; letter-spacing:.06em; font-size:9.5px; +} +/* A review row's size, in the muted voice of a footnote rather than a figure. */ +.sy-meas{font-size:11.5px; color:var(--ink-dim); font-family:var(--mono)} .sy-adv .why{color:var(--ink-2)} .sy-path{font-family:var(--mono); font-size:11px; color:var(--ink-dim); word-break:break-all; margin-top:2px} .sy-matrix{display:grid; gap:4px 6px; font-size:12px; align-items:center} diff --git a/src/lib/footprint/consumers.mjs b/src/lib/footprint/consumers.mjs new file mode 100644 index 0000000..1934a72 --- /dev/null +++ b/src/lib/footprint/consumers.mjs @@ -0,0 +1,961 @@ +// Largest consumers — the ranked "where are the bytes actually going" answer the +// System Summary strip renders (ADR-0025 §4, docs/ddd/machine-footprint.md +// "Storage breakdown"). It is a VIEW over roots this domain already has +// trust-boundary access to, not a new source: every figure is one bounded walk +// or one figure adopted from a section that already walked it. +// +// WHY THIS MODULE EXISTS RATHER THAN A SORT IN THE CLIENT. The previous ranking +// was assembled in the browser from whatever the install and storage sections +// happened to carry, which made it wrong in the one way a size ranking must +// never be wrong: it named the npx cache the machine's biggest consumer while +// ~/.npm/_cacache — three times larger — was not scanned at all, and it had no +// way to know that ~/.cache/ruvnet-brain/kb (1.9 GB) is a sixth of the cache +// root it lives in (13 GB). A ranking whose breadth is an accident of what other +// sections needed is a ranking that reads as an answer and is not one. +// +// THREE ACCOUNTING RULES, because a size ranking is trivially made dishonest: +// +// 1. CONTAINMENT. Roots nest — ~/.claude/projects is inside ~/.claude, npm's +// global root is inside mise's node install which is inside mise's installs +// tree. Bytes are counted ONCE, at the outermost row (`kind: 'root'`), and +// every enclosed row is a `kind: 'breakdown'` that explains its parent +// instead of competing with it. Containment is DERIVED from the resolved +// paths, not hand-declared, so a registry edit cannot silently start +// double-counting. Only roots are ranked and only roots are summed. +// 2. RESIDUALS. A parent with breakdowns also gets a synthesized +// `:other` row — parent minus its direct children — so the +// breakdown always adds up and "what is the rest of this?" has an answer on +// the row rather than in a user's head. This is what makes the brain row +// legible: 13 GB, of which 1.9 GB is the active KB and 11 GB is stale +// kb.bak-* copies. Merging the two figures, or reporting only the KB, are +// both ways of being wrong about the same 11 GB. +// 3. PROJECT TREES ARE OPT-IN. One repository on a working machine can be +// larger than every shared cache combined (175 GB here), and a bar chart +// containing it is a bar chart of one repository. `includeProjectTrees` +// defaults to false and the exclusion is STATED in the payload, never +// silent — an omitted category the user cannot see is its own dishonesty. +// +// Absent roots are not consumers. A cache root that does not exist on this +// machine is reported in `absent` with its path — "we looked, it is not here" — +// and kept out of the ranking and the group totals rather than ranked as a +// zero-byte consumer. Unreadable roots are reported in `unmeasured` with their +// errno and make the affected group total `partial`; they are never a zero +// (invariant 2). +// +// Metadata only (invariant 1): dirents and lstat sizes, through walk.mjs, which +// has no read path for file contents at all. +// +// KNOWN COST: this is deep-tier work. The registry names ~50 roots and several +// are enormous (a 22 GB content-addressable npm cache is ~10^5 files), so a full +// pass is tens of seconds of I/O. That is why it belongs behind the explicit +// deep scan and why `roots`/`limits` are injectable — a caller that needs a +// cheaper pass narrows the registry rather than lowering the caps, since a +// lowered cap yields a partial figure and a narrowed registry yields an honest +// smaller question. +import fs from 'node:fs'; +import path from 'node:path'; +import { + claudeDir, codexDir, configDir, globalRoot, home, isWindows, npxCacheDir, +} from '../paths.mjs'; +import { + hasValue, measured, rootMeasurements, sumMeasurements, unknown, walkTree, +} from './walk.mjs'; + +/** + * A Measurement as walk.mjs defines it: a value plus how it was obtained. + * @typedef {{ value: number|null, status: string, reason: string|null, + * asOf: number|null, partial: boolean }} Measurement + */ + +/** + * One registry entry. `path` and `match` are alternatives — a plain root, or a + * family of siblings sharing a name prefix. Containment fills in `kind` and + * `containedBy`; merging fills in `source` and `measuredBy`. + * + * @typedef {{ + * id: string, label: string, group: string, note: string, + * path?: string|null, match?: { dir: string, prefix: string }, + * allocation?: string, + * adopted?: { presence?: string, bytes: Measurement, files?: Measurement, + * newestMtimeMs?: number|null, complete?: boolean }, + * kind?: string, containedBy?: string|null, source?: string, measuredBy?: string, + * }} ConsumerDescriptor + */ + +/** + * A project the ranking may include: a ProjectFootprint from the projects + * section (whose figures are adopted) or a bare path (which is walked). + * @typedef {string|{ path: string, label?: string, presence?: string, + * totalBytes?: Measurement, treeFiles?: Measurement, + * lastActivity?: Measurement, complete?: boolean }} ProjectTreeInput + */ + +/** The ecosystem a consumer belongs to. Ecosystem rather than "kind of thing" + * because the actionable question is which toolchain is costing the disk: four + * node package caches at 5 GB each is a node answer, not four unrelated rows. + * `note` is the group's own liner note, rendered when the grouped view is on. */ +export const CONSUMER_GROUPS = Object.freeze([ + Object.freeze({ + id: 'ai-toolchain', + label: 'AI toolchain', + note: 'Local model weights, agent CLIs, their transcripts, caches and knowledge bases.', + }), + Object.freeze({ + id: 'node', + label: 'Node / npm', + note: 'Package caches, content stores and global installs for npm, pnpm, yarn and bun.', + }), + Object.freeze({ + id: 'rust', label: 'Rust', note: 'rustup toolchains and the Cargo registry/git caches.', + }), + Object.freeze({ + id: 'go', label: 'Go', note: 'The module cache and tooling caches under GOPATH.', + }), + Object.freeze({ + id: 'python', label: 'Python', note: 'pip and uv download caches and tool installs.', + }), + Object.freeze({ + id: 'java', label: 'JVM', note: 'The Maven local repository and Gradle caches/wrapper dists.', + }), + Object.freeze({ + id: 'browsers', + label: 'Browser binaries', + note: 'Playwright and Puppeteer browser downloads, re-fetched by their installers.', + }), + Object.freeze({ + id: 'system', + label: 'System & containers', + note: 'Version-manager installs, Homebrew, and container VM data.', + }), + Object.freeze({ + id: 'project-trees', + label: 'Project trees', + note: 'Working trees, .git and node_modules of the projects this machine has touched.', + }), +]); + +export const CONSUMER_GROUP_IDS = Object.freeze(CONSUMER_GROUPS.map((g) => g.id)); + +/** Twenty, because the point of the strip is breadth: at six rows the ranking + * could not show that four separate node caches outweigh the one everybody + * looks at. The panel scrolls; the payload does not need to guess how many + * fit. */ +export const CONSUMER_TOP_N = 20; + +/** How many `match`-enumerated siblings a glob row keeps as evidence paths. The + * SUM is over every match; only the sample is capped. */ +const MAX_MATCH_SAMPLES = 8; + +/** This view's own walk budget, raised from WALK_LIMITS because the defaults + * were sized for install trees and transcript roots and these roots are neither. + * Measured here: rustup exhausts the 400k entry cap at 9.4 of its 13.1 GB, and + * pnpm's content store, LM Studio and ~/.claude all bottom out on depth 16. A + * ranking whose deepest trees are systematically floors does not merely + * under-report them, it mis-ORDERS the answer — which is the exact failure this + * module exists to remove. Raised, not removed: the caps still bound the walk, + * and a root that exhausts even these still reports "≥" rather than a total. + * Costs roughly 30 s of the deep pass on this corpus; the walker's own defaults + * are unchanged for every other collector. */ +export const CONSUMER_WALK_LIMITS = Object.freeze({ + maxDepth: 24, + maxEntries: 2_000_000, +}); + +// Third-party cache roots are computed here rather than in paths.mjs on purpose: +// that module owns the kit's and its hosts' locations, and adding fifty foreign +// tools' cache conventions to it would make the kit's own path contract harder +// to audit for the sake of a read-only ranking. Kit and host paths still come +// from paths.mjs — nothing home-relative that the kit itself owns is spelled out +// below. +const xdgCache = (env) => env.XDG_CACHE_HOME || path.join(home, '.cache'); +const xdgData = (env) => env.XDG_DATA_HOME || path.join(home, '.local', 'share'); +const macCache = () => path.join(home, 'Library', 'Caches'); +const winLocalAppData = (env) => env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'); + +const at = (id, label, group, target, note) => ({ id, label, group, path: target, note }); +/** A row whose subject is a FAMILY of sibling entries — generation-numbered + * ledgers (`state_5.sqlite` and its -wal/-shm), dated backup copies + * (`kb.bak-2026-08-01`). One row per generation would be noise, and hardcoding + * today's generation would silently stop matching on the next bump. */ +const family = (id, label, group, dir, prefix, note) => ( + { id, label, group, match: { dir, prefix }, note } +); + +/** + * The curated consumer registry: pure data, no I/O, in the shape discipline of + * `src/lib/dashboard/about-directory.mjs` — authored and versioned with the + * release, joined to measurements at scan time. + * + * Platform variants are listed side by side (Playwright installs to + * ~/Library/Caches on macOS, ~/.cache on Linux, LOCALAPPDATA on Windows) rather + * than switched on `process.platform`: the wrong-platform row simply reads + * absent, and a machine that has both — a real outcome when a tool migrates its + * cache location — reports both instead of hiding one. + * + * @param {{ env?: NodeJS.ProcessEnv }} [options] + * @returns {Array} descriptors; `path` or `match`, never both + */ +export function consumerRoots({ env = process.env } = {}) { + const cache = xdgCache(env); + const data = xdgData(env); + const mac = macCache(); + const winCache = winLocalAppData(env); + const brain = path.join(cache, 'ruvnet-brain'); + const claude = claudeDir(); + const codex = codexDir(); + const mise = path.join(data, 'mise', 'installs'); + const goPath = env.GOPATH || path.join(home, 'go'); + const goModCache = env.GOMODCACHE || path.join(goPath, 'pkg', 'mod'); + const brewPrefixes = [env.HOMEBREW_PREFIX, '/opt/homebrew', '/usr/local', + '/home/linuxbrew/.linuxbrew'].filter(Boolean); + const seenBrew = new Set(); + + const rows = [ + // ── ai-toolchain ──────────────────────────────────────────────────────── + at('ollama', 'Ollama local models', 'ai-toolchain', path.join(home, '.ollama'), + 'Ollama\'s model blobs and manifests. Re-pullable: each model re-downloads on demand.'), + at('lmstudio', 'LM Studio models', 'ai-toolchain', path.join(home, '.lmstudio'), + 'LM Studio\'s downloaded weights and its own runtime. Re-downloadable per model.'), + at('huggingface', 'Hugging Face hub cache', 'ai-toolchain', path.join(cache, 'huggingface'), + 'Model and dataset snapshots pulled by any huggingface_hub client; re-downloadable.'), + at('ruvnet-brain', 'RuvNet Brain cache root', 'ai-toolchain', brain, + 'The WHOLE brain cache root, not just the active KB: the breakdown below says ' + + 'how much of it is the KB in use and how much is superseded copies.'), + at('ruvnet-brain-kb', 'RuvNet Brain active KB', 'ai-toolchain', path.join(brain, 'kb'), + 'The knowledge base the search tool actually reads.'), + family('ruvnet-brain-kb-backups', 'RuvNet Brain superseded KB copies', 'ai-toolchain', + brain, 'kb.bak', + 'Copies the installer left behind on previous updates. Nothing reads them; the ' + + 'active KB is the row above.'), + at('ruvnet-brain-models', 'RuvNet Brain embedding models', 'ai-toolchain', + path.join(brain, 'models'), + 'Embedding model files the brain loads to answer queries offline.'), + at('claude-home', 'Claude Code home', 'ai-toolchain', claude, + 'Everything under ~/.claude: transcripts, plugins, skills, snapshots and caches.'), + at('claude-transcripts', 'Claude session transcripts', 'ai-toolchain', + path.join(claude, 'projects'), + 'One .jsonl per session, per project. Historical usage reads these — deleting ' + + 'them deletes that history.'), + at('claude-plugins', 'Claude Code plugins', 'ai-toolchain', path.join(claude, 'plugins'), + 'Installed plugin repositories and their marketplace checkouts.'), + at('claude-skills', 'Claude user-scope skills', 'ai-toolchain', path.join(claude, 'skills'), + 'Skills installed at user scope, available in every project.'), + at('claude-security', 'Claude security scanner', 'ai-toolchain', + path.join(claude, 'security'), + 'The security scanner hook\'s virtualenv and its logs.'), + at('claude-file-history', 'Claude file-history snapshots', 'ai-toolchain', + path.join(claude, 'file-history'), + 'Pre-edit copies of files Claude Code changed, kept for undo.'), + at('claude-shell-snapshots', 'Claude shell snapshots', 'ai-toolchain', + path.join(claude, 'shell-snapshots'), + 'Captured shell environments, one per session start.'), + at('claude-telemetry', 'Claude telemetry', 'ai-toolchain', path.join(claude, 'telemetry'), + 'Local telemetry spool written by the CLI.'), + at('claude-image-cache', 'Claude image cache', 'ai-toolchain', + path.join(claude, 'image-cache'), 'Images pasted into sessions.'), + at('claude-todos', 'Claude todos', 'ai-toolchain', path.join(claude, 'todos'), + 'Per-session todo lists written by the CLI.'), + at('claude-statsig', 'Claude statsig', 'ai-toolchain', path.join(claude, 'statsig'), + 'Feature-flag evaluation cache written by the CLI.'), + at('claude-backups', 'Claude config backups', 'ai-toolchain', path.join(claude, 'backups'), + 'Backups of CLAUDE.md and settings taken before managed edits.'), + at('claude-history', 'Claude history.jsonl', 'ai-toolchain', + path.join(claude, 'history.jsonl'), 'The prompt-history ledger; one line per prompt.'), + at('claude-native-install', 'Claude Code native install', 'ai-toolchain', + path.join(data, 'claude'), + 'The native installer\'s versions directory — every version it has downloaded, ' + + 'not just the current one.'), + at('claude-cli-nodejs-cache', 'Claude Code per-project node cache', 'ai-toolchain', + path.join(mac, 'claude-cli-nodejs'), + 'Per-project scratch the CLI keeps outside ~/.claude (macOS cache location).'), + at('codex-home', 'Codex home', 'ai-toolchain', codex, + 'Everything under ~/.codex: rollouts, ledgers, plugin cache and snapshots.'), + at('codex-sessions', 'Codex session rollouts', 'ai-toolchain', path.join(codex, 'sessions'), + 'One rollout file per session, in dated directories. Historical usage reads these.'), + family('codex-logs-ledger', 'Codex logs ledgers', 'ai-toolchain', codex, 'logs_', + 'The logs_N.sqlite family and its -wal/-shm siblings; N bumps on schema changes ' + + 'and the old generation is left behind.'), + family('codex-state-ledger', 'Codex state ledgers', 'ai-toolchain', codex, 'state_', + 'The state_N.sqlite thread ledgers and their -wal/-shm siblings.'), + at('codex-shell-snapshots', 'Codex shell snapshots', 'ai-toolchain', + path.join(codex, 'shell_snapshots'), 'Captured shell environments, one per session.'), + at('codex-plugins', 'Codex plugin cache', 'ai-toolchain', + path.join(codex, 'plugins', 'cache'), 'Downloaded plugin payloads.'), + at('codex-cache', 'Codex cache', 'ai-toolchain', path.join(codex, 'cache'), + 'Codex\'s own scratch cache.'), + at('opencode-cache', 'OpenCode cache', 'ai-toolchain', path.join(cache, 'opencode'), + 'OpenCode\'s download and build cache.'), + at('opencode-data', 'OpenCode data root', 'ai-toolchain', path.join(data, 'opencode'), + 'OpenCode\'s session store, logs and sqlite database.'), + at('ak-config', 'agentic-kit config & indexes', 'ai-toolchain', configDir(), + 'kit.json plus the usage index, workspace store and this footprint snapshot. ' + + 'Rebuildable: every file here is derived from state the kit re-reads.'), + + // ── node ──────────────────────────────────────────────────────────────── + at('npm-cacache', 'npm content cache (_cacache)', 'node', path.join(home, '.npm', '_cacache'), + 'npm\'s content-addressable download cache. Safe to clear: npm refetches what it ' + + 'needs on the next install.'), + at('npx-cache', 'npx cache envs', 'node', npxCacheDir(), + 'One throwaway install tree per `npx ` invocation. Reproducible; stale envs ' + + 'are listed as reclaimables in the Storage view.'), + at('npm-global-root', 'npm global node_modules', 'node', safeGlobalRoot(), + 'Every globally installed npm package, including the managed tools — `npm root -g`.'), + at('pnpm-store-xdg', 'pnpm content store (XDG)', 'node', path.join(data, 'pnpm', 'store'), + 'pnpm\'s content-addressable store; project node_modules hard-link into it.'), + at('pnpm-store-library', 'pnpm content store (older layout)', 'node', + path.join(home, 'Library', 'pnpm', 'store'), + 'The pre-XDG pnpm store location. Present here means an older pnpm wrote it; it is ' + + 'not linked by projects the current pnpm installs.'), + at('pnpm-store-legacy', 'pnpm store (legacy ~/.pnpm-store)', 'node', + path.join(home, '.pnpm-store'), 'The oldest pnpm store location.'), + at('pnpm-cache', 'pnpm metadata cache', 'node', path.join(cache, 'pnpm'), + 'Registry metadata and side caches pnpm keeps outside the content store.'), + at('mise-node', 'mise node runtimes', 'node', path.join(mise, 'node'), + 'Every Node version mise has installed, each with its own global lib tree.'), + at('node-gyp-cache', 'node-gyp headers cache', 'node', path.join(mac, 'node-gyp'), + 'Node headers and libs per version, downloaded to build native addons.'), + at('node-gyp-cache-xdg', 'node-gyp headers cache', 'node', path.join(cache, 'node-gyp'), + 'Node headers and libs per version, downloaded to build native addons.'), + at('yarn-cache', 'Yarn cache', 'node', path.join(mac, 'Yarn'), + 'Yarn\'s package cache; re-fetched on demand.'), + at('yarn-cache-xdg', 'Yarn cache', 'node', path.join(cache, 'yarn'), + 'Yarn\'s package cache; re-fetched on demand.'), + at('bun', 'Bun home', 'node', path.join(home, '.bun'), + 'The bun runtime, its global installs and its package cache.'), + + // ── rust ──────────────────────────────────────────────────────────────── + at('rustup', 'rustup toolchains', 'rust', path.join(home, '.rustup'), + 'Every installed Rust toolchain and its docs/std components.'), + at('cargo', 'Cargo home', 'rust', path.join(home, '.cargo'), + 'Cargo\'s registry cache, git checkouts and installed binaries.'), + at('cargo-registry', 'Cargo registry cache', 'rust', path.join(home, '.cargo', 'registry'), + 'Downloaded crate sources and their .crate archives; re-fetchable.'), + + // ── go ────────────────────────────────────────────────────────────────── + at('go-modcache', 'Go module cache', 'go', goModCache, + 'Extracted module sources; `go clean -modcache` re-downloads them.'), + at('goimports-cache', 'goimports cache', 'go', path.join(mac, 'goimports'), + 'Index goimports keeps to resolve packages quickly.'), + + // ── python ────────────────────────────────────────────────────────────── + at('uv-cache', 'uv cache', 'python', path.join(cache, 'uv'), + 'uv\'s wheel and source distribution cache; re-fetchable.'), + at('uv-data', 'uv tool installs', 'python', path.join(data, 'uv'), + 'Environments uv created for `uv tool install`.'), + at('pip-cache', 'pip cache', 'python', path.join(mac, 'pip'), + 'pip\'s HTTP and wheel cache; re-fetchable.'), + at('pip-cache-xdg', 'pip cache', 'python', path.join(cache, 'pip'), + 'pip\'s HTTP and wheel cache; re-fetchable.'), + + // ── java ──────────────────────────────────────────────────────────────── + at('maven-repo', 'Maven local repository', 'java', path.join(home, '.m2', 'repository'), + 'Every artifact Maven or Gradle resolved into ~/.m2; re-resolvable from remotes.'), + at('gradle-home', 'Gradle home', 'java', path.join(home, '.gradle'), + 'Gradle\'s build caches, resolved dependencies and downloaded wrapper distributions.'), + + // ── browsers ──────────────────────────────────────────────────────────── + at('playwright-mac', 'Playwright browsers', 'browsers', path.join(mac, 'ms-playwright'), + 'Browser builds downloaded by `playwright install` (macOS location).'), + at('playwright-xdg', 'Playwright browsers', 'browsers', path.join(cache, 'ms-playwright'), + 'Browser builds downloaded by `playwright install` (Linux/XDG location).'), + at('playwright-win', 'Playwright browsers', 'browsers', path.join(winCache, 'ms-playwright'), + 'Browser builds downloaded by `playwright install` (Windows location).'), + at('puppeteer', 'Puppeteer browsers', 'browsers', path.join(cache, 'puppeteer'), + 'Chrome builds downloaded by Puppeteer\'s installer.'), + + // ── system ────────────────────────────────────────────────────────────── + at('mise-installs', 'mise toolchain installs', 'system', mise, + 'Every runtime mise manages — node, python, go, java — all versions kept.'), + { + ...at('docker-data', 'Docker Desktop VM data', 'system', + path.join(home, 'Library', 'Containers', 'com.docker.docker', 'Data'), + 'The Linux VM disk image backing Docker Desktop: images, volumes and layers live ' + + 'inside it, so it does not shrink when you delete images. Counted as blocks ' + + 'actually written — the image is sparse, and its apparent size is far larger.'), + allocation: 'blocks', + }, + at('docker-config', 'Docker CLI config', 'system', path.join(home, '.docker'), + 'CLI config, contexts and credential helpers.'), + at('homebrew-cache', 'Homebrew download cache', 'system', path.join(mac, 'Homebrew'), + 'Downloaded bottles and source tarballs; `brew cleanup` removes them.'), + at('homebrew-cache-xdg', 'Homebrew download cache', 'system', path.join(cache, 'Homebrew'), + 'Downloaded bottles and source tarballs; `brew cleanup` removes them.'), + ]; + + for (const prefix of brewPrefixes) { + const cellar = path.join(prefix, 'Cellar'); + if (seenBrew.has(cellar)) continue; + seenBrew.add(cellar); + rows.push(at(`homebrew-cellar:${prefix}`, 'Homebrew Cellar', 'system', cellar, + `Installed formulae under ${prefix}, all versions kept until \`brew cleanup\`.`)); + } + if (isWindows) { + rows.push(at('npm-cache-win', 'npm cache (Windows)', 'node', + path.join(winCache, 'npm-cache', '_cacache'), + 'npm\'s content-addressable download cache. Safe to clear; npm refetches.')); + } + return rows; +} + +/** `npm root -g` throws when npm is absent (paths.mjs). A machine without npm + * is a legitimate machine, and the row simply reads absent rather than taking + * the registry down with it. */ +function safeGlobalRoot() { + try { return globalRoot(); } catch { return null; } +} + +// Path comparison is case-insensitive off Linux because APFS and NTFS are: on +// macOS ~/Library/Caches and ~/library/caches are the same directory, and a +// case-sensitive containment test would rank a child as its own root and +// double-count it. +const foldCase = process.platform !== 'linux'; +const normalizePath = (p) => { + const abs = path.resolve(p); + return foldCase ? abs.toLowerCase() : abs; +}; + +/** Is `child` strictly inside `parent`? */ +export function isInside(parent, child) { + if (!parent || !child) return false; + const a = normalizePath(parent); + const b = normalizePath(child); + if (a === b) return false; + const rel = path.relative(a, b); + return Boolean(rel) && !rel.startsWith('..') && !path.isAbsolute(rel); +} + +/** The path a descriptor is accounted at. A `match` family anchors at + * `/` — a pseudo-path, deliberately: the family's members live + * INSIDE `dir`, so anchoring at `dir` itself would make the family a sibling of + * its own container and let both be ranked, double-counting every byte. The + * pseudo-path cannot capture the real siblings it is derived from either + * (`kb.bak` is not an ancestor of `kb.bak-2026-08-01`), so it nests one way + * only, which is exactly what containment needs. */ +const anchorOf = (desc) => ( + desc.path ?? (desc.match ? path.join(desc.match.dir, desc.match.prefix) : null) +); + +/** + * Derive containment across descriptors. Each row is tagged `kind: 'root'` (its + * bytes are counted once, here) or `kind: 'breakdown'` with `containedBy` naming + * the nearest enclosing row. Nearest, not first: mise installs ⊃ mise node ⊃ npm + * global root must chain, so the residual of each parent is over its DIRECT + * children only. + * + * Pure over the descriptor list — no filesystem access — so the accounting is + * testable without a machine to measure. + * + * @param {Array} descriptors + * @returns {Array} the same descriptors, `kind` and + * `containedBy` filled in + */ +export function assignContainment(descriptors) { + const rows = descriptors.filter((d) => anchorOf(d)); + return rows.map((desc) => { + const mine = anchorOf(desc); + let parent = null; + for (const other of rows) { + if (other === desc) continue; + const theirs = anchorOf(other); + if (!isInside(theirs, mine)) continue; + if (!parent || isInside(anchorOf(parent), theirs)) parent = other; + } + return { ...desc, kind: parent ? 'breakdown' : 'root', containedBy: parent?.id ?? null }; + }); +} + +/** Sum a `match` family: every immediate entry of `dir` whose name starts with + * `prefix`. An unreadable directory is unknown-with-errno; a readable one with + * no matches is a real, measured zero and reads as absent — nothing named that + * is there. */ +function measureFamily(desc, { walk, limits, asOf, fsImpl }) { + const { dir, prefix } = desc.match; + let entries; + try { + // One non-recursive listing to find the members; every member is then handed + // to the shared walker, which keeps the caps, the symlink rule and the + // degrade-this-node-only behaviour where they belong. This is the same + // enumerate-then-walk shape install.mjs's npxEnvNodes already uses. + entries = fsImpl.readdirSync(dir, { withFileTypes: true }); + } catch (err) { + const code = err?.code || 'io'; + if (code === 'ENOENT') { + return { + presence: 'absent', bytes: measured(0, { asOf }), files: measured(0, { asOf }), + newestMtimeMs: null, matchedPaths: [], matchedCount: 0, complete: true, + }; + } + return { + presence: 'degraded', bytes: unknown(code), files: unknown(code), + newestMtimeMs: null, matchedPaths: [], matchedCount: 0, complete: false, + }; + } + const byteParts = []; + const fileParts = []; + const samples = []; + let matched = 0; + let newest = null; + let complete = true; + for (const entry of entries) { + if (!entry.name.startsWith(prefix) || entry.isSymbolicLink()) continue; + matched += 1; + const target = path.join(dir, entry.name); + if (samples.length < MAX_MATCH_SAMPLES) samples.push(target); + const result = walk(target, { ...limits, fsImpl }); + const node = rootMeasurements(result, { asOf }); + byteParts.push(node.bytes); + fileParts.push(node.files); + if (result.newestMtimeMs !== null && (newest === null || result.newestMtimeMs > newest)) { + newest = result.newestMtimeMs; + } + if (result.complete === false) complete = false; + } + if (!matched) { + return { + presence: 'absent', bytes: measured(0, { asOf }), files: measured(0, { asOf }), + newestMtimeMs: null, matchedPaths: [], matchedCount: 0, complete: true, + }; + } + return { + presence: 'present', + bytes: sumMeasurements(byteParts, { asOf }), + files: sumMeasurements(fileParts, { asOf }), + newestMtimeMs: newest, + matchedPaths: samples, + matchedCount: matched, + complete, + }; +} + +/** + * Bytes a root really OCCUPIES, from allocated blocks rather than apparent + * size. Only for roots declared `allocation: 'blocks'`, and only because + * apparent size is catastrophically wrong for them: Docker Desktop's VM image is + * a sparse file whose apparent size on this machine is ~4 TB against ~15 GB + * actually written, which would not merely mis-order the ranking, it would put a + * figure larger than the disk at the top of it. + * + * The walker still owns the traversal; the extra lstat per file is what buys + * `blocks`, which `onFile` does not carry. A platform that reports no block + * count (Windows) falls back to that file's apparent size, so the row degrades + * to the ordinary basis rather than to zero. + */ +function measureAllocated(desc, { walk, limits, asOf, fsImpl }) { + let allocated = 0; + let apparent = 0; + let files = 0; + let newest = null; + let estimated = 0; + const result = walk(desc.path, { + ...limits, + fsImpl, + onFile: ({ file, bytes, mtimeMs }) => { + apparent += bytes; + files += 1; + if (newest === null || mtimeMs > newest) newest = mtimeMs; + let blocks; + try { blocks = Number(fsImpl.lstatSync(file).blocks); } catch { blocks = null; } + // Zero blocks is a valid answer (an empty file, or one held entirely in + // an inode); only a platform that reports no block count at all falls + // back to apparent size, and that fallback is counted so the row can say + // its basis is mixed. + if (Number.isFinite(blocks) && blocks >= 0) allocated += blocks * 512; + else { allocated += bytes; estimated += 1; } + }, + }); + const node = rootMeasurements(result, { asOf }); + if (node.presence !== 'present') { + return { + presence: node.presence, bytes: node.bytes, files: node.files, + newestMtimeMs: null, matchedPaths: [], matchedCount: null, + apparentBytes: node.bytes, basis: 'allocated-blocks', complete: node.presence === 'absent', + }; + } + const partial = result.complete === false; + return { + presence: 'present', + bytes: measured(allocated, { asOf, partial }), + files: measured(files, { asOf, partial }), + newestMtimeMs: newest, + matchedPaths: [], + matchedCount: null, + apparentBytes: measured(apparent, { asOf, partial }), + basis: estimated ? 'allocated-blocks (partly apparent)' : 'allocated-blocks', + complete: !partial, + }; +} + +function measureDescriptor(desc, ctx) { + // An adopted figure is already measured — re-walking a 22 GB tree a second + // time to learn what another section already knows is pure I/O cost. + if (desc.adopted) { + return { + presence: desc.adopted.presence ?? (hasValue(desc.adopted.bytes) ? 'present' : 'degraded'), + bytes: desc.adopted.bytes, + files: desc.adopted.files ?? unknown('file count not carried by the adopted figure'), + newestMtimeMs: desc.adopted.newestMtimeMs ?? null, + matchedPaths: [], + matchedCount: null, + basis: 'adopted', + complete: desc.adopted.complete !== false, + }; + } + if (desc.match) return measureFamily(desc, ctx); + if (desc.allocation === 'blocks') return measureAllocated(desc, ctx); + const result = ctx.walk(desc.path, { ...ctx.limits, fsImpl: ctx.fsImpl }); + const node = rootMeasurements(result, { asOf: ctx.asOf }); + return { + presence: node.presence, + bytes: node.bytes, + files: node.files, + newestMtimeMs: result.newestMtimeMs ?? null, + matchedPaths: [], + matchedCount: null, + basis: 'apparent-size', + // An absent root is a COMPLETE measurement of a real zero, not an + // incomplete one: the walker reports `complete: false` for it only because + // it could not read a directory that is not there. + complete: node.presence === 'absent' || result.complete !== false, + }; +} + +/** The ecosystem of each install-section shared cache. Kept beside the adoption + * code rather than in install.mjs: the ecosystem grouping is this view's + * vocabulary, and install.mjs has no opinion about it. */ +const SHARED_CACHE_GROUPS = Object.freeze({ + 'npx-envs': 'node', + 'claude-plugins': 'ai-toolchain', + 'codex-plugins': 'ai-toolchain', + playwright: 'browsers', + 'playwright-mac': 'browsers', + 'playwright-win': 'browsers', + puppeteer: 'browsers', +}); + +/** Descriptors adopted from the install section: managed-tool trees and the + * shared caches it already walked. Their figures are reused as-is; where the + * registry already names the same path, the registry's label and note win and + * only the MEASUREMENT is taken (one walk, one row, two sources agreeing). */ +export function installDescriptors(install) { + if (!install) return []; + const rows = []; + for (const tool of install.tools ?? []) { + if (!tool?.root || !hasValue(tool.bytes)) continue; + rows.push({ + id: `install:${tool.tool}`, + label: `${tool.label} install tree`, + group: 'ai-toolchain', + path: tool.root, + note: `The ${tool.label} install tree${tool.version ? ` (v${tool.version})` : ''}, ` + + 'measured by the Install section.', + adopted: { + presence: 'present', bytes: tool.bytes, files: tool.files, + newestMtimeMs: tool.newestMtimeMs ?? null, complete: tool.complete !== false, + }, + }); + } + for (const cache of install.sharedCaches ?? []) { + if (!cache?.path) continue; + rows.push({ + id: `cache:${cache.id}`, + label: cache.label, + group: SHARED_CACHE_GROUPS[cache.id] ?? 'system', + path: cache.path, + note: 'Shared cache measured by the Install section.', + adopted: { + presence: cache.presence, bytes: cache.bytes, files: cache.files, + newestMtimeMs: cache.newestMtimeMs ?? null, complete: cache.complete !== false, + }, + }); + } + return rows; +} + +/** + * Project working trees as consumer descriptors. A ProjectFootprint from the + * projects section already carries `totalBytes` (tree + .git + node_modules), so + * inclusion costs nothing extra; a bare `{ path, label }` is walked. + * + * @param {Array} projects + * @returns {Array} + */ +export function projectTreeDescriptors(projects) { + const rows = []; + for (const project of projects ?? []) { + const footprint = typeof project === 'string' ? null : project; + const target = typeof project === 'string' ? project : project?.path; + if (!target) continue; + const total = footprint?.totalBytes ?? null; + const adopted = hasValue(total) && footprint + ? { + presence: footprint.presence ?? 'present', + bytes: total, + files: footprint.treeFiles, + newestMtimeMs: hasValue(footprint.lastActivity) ? footprint.lastActivity.value : null, + complete: footprint.complete !== false, + } + : null; + rows.push({ + id: `project:${target}`, + label: footprint?.label ?? path.basename(target), + group: 'project-trees', + path: target, + note: adopted + ? 'Working tree plus .git plus node_modules, measured by the Projects scan.' + : 'Working tree as walked here; .git and node_modules included.', + ...(adopted ? { adopted } : {}), + }); + } + return rows; +} + +/** Merge descriptor sources, path-keyed. The first source to claim a path owns + * its identity (label, note, group); later sources contribute only a + * measurement it lacks. That is what keeps the registry's editorial notes while + * still reusing the install section's walk. */ +function mergeDescriptors(sources) { + const byPath = new Map(); + const out = []; + for (const { source, rows } of sources) { + for (const desc of rows) { + const anchor = anchorOf(desc); + if (!anchor) continue; + const key = normalizePath(anchor) + (desc.match ? `::${desc.match.prefix}` : ''); + const existing = byPath.get(key); + if (existing) { + if (!existing.adopted && desc.adopted) { + existing.adopted = desc.adopted; + existing.measuredBy = source; + } + continue; + } + const row = { ...desc, source, measuredBy: desc.adopted ? source : 'consumers' }; + byPath.set(key, row); + out.push(row); + } + } + return out; +} + +/** Parent minus its DIRECT children. The row exists so a breakdown always adds + * up: without it, "13 GB, of which 1.9 GB is the KB" invites the reader to + * invent the missing 11 GB. Unknown inputs make it unknown — never a + * difference computed against a fabricated zero — and a negative difference + * (children overlapping in a way containment did not catch) reports itself + * rather than rendering as a plausible small number. */ +function residualRow(parent, children, asOf) { + const parts = children.map((c) => c.bytes); + if (!hasValue(parent.bytes) || parts.some((m) => !hasValue(m))) { + return { + bytes: unknown('parent or a breakdown row is unmeasured'), + files: unknown('parent or a breakdown row is unmeasured'), + negative: false, + }; + } + const bytes = parent.bytes.value - parts.reduce((acc, m) => acc + m.value, 0); + if (bytes < 0) { + return { + bytes: unknown('breakdown rows sum to more than their parent'), + files: unknown('breakdown rows sum to more than their parent'), + negative: true, + }; + } + const partial = parent.bytes.partial === true || parts.some((m) => m.partial === true); + const files = hasValue(parent.files) && children.every((c) => hasValue(c.files)) + ? measured( + Math.max(0, parent.files.value - children.reduce((acc, c) => acc + c.files.value, 0)), + { asOf, partial }, + ) + : unknown('file counts not available for every breakdown row'); + return { bytes: measured(bytes, { asOf, partial }), files, negative: false }; +} + +const rankValue = (row) => (hasValue(row.bytes) ? row.bytes.value : -1); + +/** + * The ranked-consumers view. + * + * @param {{ + * now?: () => number, walk?: typeof walkTree, limits?: object, + * roots?: Array|null, env?: NodeJS.ProcessEnv, + * install?: object|null, projects?: Array|null, + * includeProjectTrees?: boolean, topN?: number, + * extraRoots?: Array, + * fsImpl?: typeof fs, + * }} [options] `roots` replaces the registry outright (tests, narrowed scans); + * `extraRoots` adds to it. `install` and `projects` are already-collected + * sections whose figures are adopted rather than re-walked. + * @returns {{ + * asOf: number, includeProjectTrees: boolean, topN: number, + * rows: object[], top: object[], groups: object[], totals: object, + * absent: object[], unmeasured: object[], projectTrees: object, + * accounting: object, complete: boolean, + * }} + */ +export function collectConsumers({ + now = Date.now, + walk = walkTree, + limits = {}, + roots = null, + env = process.env, + install = null, + projects = null, + includeProjectTrees = false, + topN = CONSUMER_TOP_N, + extraRoots = [], + fsImpl = fs, +} = {}) { + const asOf = now(); + const ctx = { walk, limits: { ...CONSUMER_WALK_LIMITS, ...limits }, asOf, fsImpl }; + const candidates = Array.isArray(projects) ? projects : []; + + const descriptors = assignContainment(mergeDescriptors([ + { source: 'registry', rows: roots ?? consumerRoots({ env }) }, + { source: 'install', rows: installDescriptors(install) }, + { source: 'caller', rows: extraRoots }, + { + source: 'projects', + rows: includeProjectTrees ? projectTreeDescriptors(candidates) : [], + }, + ])); + + const measuredRows = descriptors.map((desc) => { + const m = measureDescriptor(desc, ctx); + return { + id: desc.id, + label: desc.label, + path: desc.path ?? null, + pathPattern: desc.match ? path.join(desc.match.dir, `${desc.match.prefix}*`) : null, + matchedPaths: m.matchedPaths, + matchedCount: m.matchedCount, + group: desc.group, + kind: desc.kind, + containedBy: desc.containedBy, + presence: m.presence, + bytes: m.bytes, + files: m.files, + // How the bytes were obtained. 'apparent-size' is every ordinary row; + // 'allocated-blocks' rows also carry `apparentBytes`, and the gap between + // the two is the whole reason that basis exists. + basis: m.basis ?? 'apparent-size', + apparentBytes: m.apparentBytes ?? null, + newestMtimeMs: m.newestMtimeMs, + accountingNote: desc.note, + source: desc.source, + measuredBy: desc.measuredBy, + residual: false, + complete: m.complete, + }; + }); + + const byId = new Map(measuredRows.map((row) => [row.id, row])); + const directChildren = new Map(); + for (const row of measuredRows) { + if (!row.containedBy || !byId.has(row.containedBy)) continue; + const list = directChildren.get(row.containedBy) ?? []; + list.push(row); + directChildren.set(row.containedBy, list); + } + + const residuals = []; + for (const [parentId, children] of directChildren) { + const parent = byId.get(parentId); + if (parent.presence === 'absent') continue; + const { bytes, files } = residualRow(parent, children, asOf); + residuals.push({ + id: `${parentId}:other`, + label: `everything else under ${parent.label}`, + path: parent.path, + pathPattern: null, + matchedPaths: [], + matchedCount: null, + group: parent.group, + kind: 'breakdown', + containedBy: parentId, + presence: parent.presence, + bytes, + files, + basis: 'derived', + apparentBytes: null, + newestMtimeMs: null, + accountingNote: `${parent.label} minus the ${children.length} row(s) broken out of it, ` + + 'so the breakdown adds up to the parent.', + source: 'derived', + measuredBy: 'consumers', + residual: true, + complete: parent.complete, + }); + } + + const rows = [...measuredRows, ...residuals] + .sort((a, b) => rankValue(b) - rankValue(a)); + + // Totals sum every ROOT, absent ones included: an absent root's measured zero + // is a real zero and adding it changes nothing. Ranking, by contrast, drops + // them — a directory that is not there is not a consumer, and listing it at + // "0 B" is the shape of an unknown wearing a number. + const allRoots = rows.filter((row) => row.kind === 'root'); + const ranked = allRoots.filter((row) => row.presence !== 'absent' && hasValue(row.bytes)); + const groups = CONSUMER_GROUPS.map((group) => { + const members = allRoots.filter((row) => row.group === group.id); + const empty = group.id === 'project-trees' && !includeProjectTrees + ? unknown('project trees not measured: the toggle is off') + : measured(0, { asOf }); + return { + ...group, + rowCount: members.length, + bytes: members.length ? sumMeasurements(members.map((r) => r.bytes), { asOf }) : empty, + files: members.length ? sumMeasurements(members.map((r) => r.files), { asOf }) : empty, + largest: members.filter((r) => hasValue(r.bytes)) + .sort((a, b) => rankValue(b) - rankValue(a))[0]?.id ?? null, + }; + }).sort((a, b) => (b.bytes.value ?? -1) - (a.bytes.value ?? -1)); + + const absent = rows.filter((row) => row.presence === 'absent') + .map(({ id, label, path: rootPath, group, kind }) => ({ id, label, path: rootPath, group, kind })); + const unmeasured = rows.filter((row) => row.presence === 'degraded') + .map(({ id, label, path: rootPath, group, kind, bytes }) => ( + { id, label, path: rootPath, group, kind, reason: bytes.reason } + )); + + return { + asOf, + includeProjectTrees, + topN, + rows, + top: ranked.slice(0, topN), + groups, + totals: { + bytes: sumMeasurements(allRoots.map((r) => r.bytes), { asOf }), + files: sumMeasurements(allRoots.map((r) => r.files), { asOf }), + rootCount: measured(allRoots.length, { asOf }), + breakdownCount: measured(rows.filter((r) => r.kind === 'breakdown').length, { asOf }), + rankedCount: measured(ranked.length, { asOf }), + }, + absent, + unmeasured, + projectTrees: { + included: includeProjectTrees, + candidates: candidates.length, + // Stated rather than silent: an excluded category the reader cannot see is + // the same failure as an unknown rendered as zero. + reason: includeProjectTrees + ? null + : 'Project working trees are excluded by default: a single large repository can ' + + 'outweigh every shared cache combined and flatten the ranking.', + }, + accounting: { + basis: 'One bounded walk per root (symlinks never followed, so a symlinked tree is ' + + 'counted where it really lives), or the figure the Install/Projects scan already ' + + 'measured for that exact path.', + containment: 'Nested roots are counted once, at the outermost row. Rows inside another ' + + 'row are breakdowns and are excluded from the ranking and the group totals.', + residuals: 'Every row with breakdowns also carries an "everything else" row, so a ' + + 'breakdown always sums to its parent.', + absent: 'Roots that do not exist on this machine are listed as absent, not ranked as ' + + 'zero-byte consumers.', + projectTrees: 'Project working trees join the ranking only when the toggle is on.', + }, + complete: rows.every((row) => row.complete !== false) && !unmeasured.length, + }; +} diff --git a/src/lib/footprint/index.mjs b/src/lib/footprint/index.mjs index de8474f..e869027 100644 --- a/src/lib/footprint/index.mjs +++ b/src/lib/footprint/index.mjs @@ -8,7 +8,8 @@ // TTL-cached ~60s in memory following buildProjectSnapshotCache's // pattern in dashboard-server.mjs — machine-wide data, one entry per // collector instance, no per-caller key. -// DEEP the full storage walk + per-project LOC + cross-host catalog dedup. +// DEEP the full storage walk + per-project LOC and stack detection + +// cross-host catalog dedup + the ranked largest-consumers view. // Explicit, user-triggered, SINGLE-FLIGHT: a second request attaches // to the running scan instead of racing it (invariant 7). usage-index // keys its coalescing map by the options that change the RESULT; a @@ -32,7 +33,6 @@ import { claudeDir, claudeSettingsPath, claudeUserMcpPath, codexConfigPath, codexDir, configDir, home, } from '../paths.mjs'; import { loadKitConfig } from '../config.mjs'; -import { discoverRuvfloProjects } from '../dashboard/project-discovery.mjs'; import { defaultOpencodeDbPath } from '../usage-opencode.mjs'; import { UNKNOWN, measured, statNode, unknown } from './walk.mjs'; import { collectRuntimeCensus } from './runtime.mjs'; @@ -40,6 +40,8 @@ import { collectInstall } from './install.mjs'; import { collectStorage } from './storage.mjs'; import { collectCatalog } from './catalog.mjs'; import { collectProjects } from './projects.mjs'; +import { collectConsumers } from './consumers.mjs'; +import { discoverProjectSources } from './project-sources.mjs'; import { SNAPSHOT_SECTIONS, SNAPSHOT_STALE_AFTER_MS, carryForward, readSnapshot, snapshotFreshness, snapshotPath, summarizeCompleteness, writeSnapshot, @@ -52,9 +54,16 @@ export const CHEAP_TTL_MS = 60_000; /** Deep-scan progress phases, in order. `idle` is the state before any scan * has run in this process; `done`/`failed` are terminal. */ export const SCAN_PHASES = Object.freeze([ - 'idle', 'install', 'storage', 'catalog', 'projects', 'persist', 'done', 'failed', + 'idle', 'install', 'storage', 'catalog', 'projects', 'consumers', 'persist', 'done', 'failed', ]); +/** Whether the ranked-consumers view walks project working trees. Off by + * default and deliberately: one repository on this machine is 175 GB, which is + * larger than every shared cache combined, so a ranking containing it is a + * ranking of that one repository. The exclusion is stated in the payload + * (`consumers.projectTrees.reason`), never silent. */ +export const INCLUDE_PROJECT_TREES_DEFAULT = false; + /** Individually-known files the cheap tier can stat without a walk. Each is one * lstat, so the whole list is affordable on every request — which is the * point: these are the files that grow fastest between deep scans (ledgers, @@ -142,6 +151,37 @@ function freshScanState() { durationMs: null, error: null, asOf: null, + // What the RUNNING scan is measuring, not what the next one would: `null` + // until a scan starts, so "no scan has run" cannot read as "trees off". + includeProjectTrees: null, + }; +} + +/** + * Read a discovery result without deciding what it means for the caller. + * + * Two shapes are legitimate. `discoverProjectSources()` returns a payload whose + * `projects` include projects that have since been DELETED — that is the point + * of `everSeen` — so only the surviving subset can be measured, and the payload + * itself is forwarded so the Projects section can publish everSeen/onDisk/ + * gitRepos/method rather than recomputing them from the rows it kept. A bare + * array is an explicit catalog and is passed straight through as one. + * + * @param {any} result a discoverProjectSources() payload, an explicit catalog + * array, or whatever an injected discovery returned — including nothing + * @returns {{ sources: object|null, catalog: Array|null, onDisk: Array }} + */ +function readDiscovery(result) { + if (Array.isArray(result)) { + return { sources: null, catalog: result, onDisk: result.filter((p) => p?.path) }; + } + if (!result || typeof result !== 'object' || !Array.isArray(result.projects)) { + return { sources: null, catalog: null, onDisk: [] }; + } + return { + sources: result, + catalog: null, + onDisk: result.projects.filter((project) => project?.path && project.exists !== false), }; } @@ -154,11 +194,18 @@ function freshScanState() { * without touching the real machine — the same discipline the individual * collectors already follow. * + * `discoverProjects` supplies candidate paths only (invariant 9). It returns a + * discoverProjectSources() payload — every project ANY host has ever recorded a + * session in, which is what the Projects KPI means; an array is still accepted + * and taken as an explicit catalog, the shape the older ruflo-state discovery + * returned. + * * @param {{ * now?: () => number, ttlMs?: number, staleAfterMs?: number, * snapshotFile?: string, fsImpl?: typeof fs, cwd?: string, * loadConfig?: () => object, - * discoverProjects?: () => Array<{ path: string, label: string }>, + * discoverProjects?: (options?: object) => object|Array<{ path: string, label: string }>, + * includeProjectTrees?: boolean, * collectors?: Record, collectorOptions?: Record, * readSnapshotImpl?: typeof readSnapshot, writeSnapshotImpl?: typeof writeSnapshot, * }} [options] @@ -171,7 +218,8 @@ export function createSystemCollector({ fsImpl = fs, cwd = process.cwd(), loadConfig = loadKitConfig, - discoverProjects = discoverRuvfloProjects, + discoverProjects = discoverProjectSources, + includeProjectTrees: includeProjectTreesDefault = INCLUDE_PROJECT_TREES_DEFAULT, collectors = {}, collectorOptions = {}, readSnapshotImpl = readSnapshot, @@ -183,6 +231,7 @@ export function createSystemCollector({ storage: collectStorage, catalog: collectCatalog, projects: collectProjects, + consumers: collectConsumers, ...collectors, }; const snapshotOpts = { ...(snapshotFile ? { file: snapshotFile } : {}), fsImpl }; @@ -194,6 +243,10 @@ export function createSystemCollector({ /** @type {Promise|null} — the single-flight slot (invariant 7). */ let inFlight = null; let scan = freshScanState(); + /** Sticky across scans, because the chip that sets it reads its own state + * back off the LAST scan's payload: a plain rescan that silently reverted to + * the default would flip a control the user did not touch. */ + let includeProjectTrees = includeProjectTreesDefault === true; const markPhase = (phase, extra = {}) => { scan = { ...scan, phase, ...extra }; @@ -262,14 +315,23 @@ export function createSystemCollector({ }; } - /** Run the four deep collectors in order, persist, invalidate the cheap + /** Run the five deep collectors in order, persist, invalidate the cheap * cache. Never rejects: a scan that blows up records the reason in `scan` * and resolves with `ok: false`, so a fire-and-forget caller (the HTTP * route) cannot produce an unhandled rejection and a waiting caller (the * CLI) still gets an answer. */ async function runDeep() { const startedAt = now(); - scan = { ...freshScanState(), running: true, phase: 'install', startedAt, asOf: startedAt }; + const withTrees = includeProjectTrees; + scan = { + ...freshScanState(), + running: true, + phase: 'install', + startedAt, + asOf: startedAt, + includeProjectTrees: withTrees, + }; + /** @type {Record} */ const sections = {}; try { // Yield BEFORE the first synchronous collector. `refreshDeep()` runs this @@ -278,12 +340,18 @@ export function createSystemCollector({ // start-or-attach route must return while the scan runs, not after it. await breathe(); - // Discovery is a candidate-path source shared by two collectors - // (invariant 9). Resolve it ONCE so storage's learning-store nodes and - // the Projects table describe the same set of projects. - let catalog = null; - try { catalog = discoverProjects(); } catch { catalog = null; } - const projectPaths = Array.isArray(catalog) ? catalog.map((p) => p.path) : null; + // Discovery is a candidate-path source shared by three collectors + // (invariant 9). Resolve it ONCE — a ~3,200-transcript sweep — so + // storage's learning-store nodes, the Projects table and the consumers + // ranking describe the same set of projects. + let discovered = null; + try { discovered = discoverProjects({ fsImpl }); } catch { discovered = null; } + const { sources, catalog, onDisk } = readDiscovery(discovered); + // Only projects that still exist can be walked; the vanished ones survive + // in the Projects section's everSeen, not as unmeasurable paths handed to + // a collector. `null` (discovery failed) stays null — storage reports that + // as unknown, which a zero-length array would not. + const projectPaths = discovered ? onDisk.map((project) => project.path) : null; let cfg = {}; try { cfg = loadConfig() ?? {}; } catch { cfg = {}; } @@ -303,8 +371,14 @@ export function createSystemCollector({ sections.catalog = collect.catalog({ cwd, cfg, now: () => startedAt, fsImpl, ...(collectorOptions.catalog ?? {}) }); await breathe(); - markPhase('projects', { scanned: 0, total: Array.isArray(catalog) ? catalog.length : 0 }); + markPhase('projects', { scanned: 0, total: onDisk.length }); sections.projects = collect.projects({ + // Whichever shape discovery produced: a sources payload carries + // everSeen/onDisk/gitRepos/method through to the KPIs, an explicit + // array is measured verbatim. Both slots are null when discovery + // failed, which makes the section report the failure rather than an + // empty machine. + sources, projects: catalog, now: () => startedAt, fsImpl, @@ -315,6 +389,28 @@ export function createSystemCollector({ }); await breathe(); + // Consumers runs LAST because it is the only collector that can adopt + // another section's figures instead of re-walking: install's tool trees + // and — when project trees are in scope — every ProjectFootprint's + // totalBytes, which is what keeps the toggle from walking a 175 GB + // repository a second time in the same scan. + markPhase('consumers'); + const measuredProjects = sections.projects?.projects; + sections.consumers = collect.consumers({ + now: () => startedAt, + fsImpl, + install: sections.install ?? null, + // Adopted footprints when the Projects phase measured any, the bare + // discovered paths otherwise (a truncated or LOC-less projects scan + // still leaves the ranking able to walk what it names). + projects: Array.isArray(measuredProjects) && measuredProjects.length + ? measuredProjects + : projectPaths, + includeProjectTrees: withTrees, + ...(collectorOptions.consumers ?? {}), + }); + await breathe(); + markPhase('persist'); const persisted = writeSnapshotImpl(sections, { ...snapshotOpts, now: now(), asOf: startedAt }); const finishedAt = now(); @@ -366,10 +462,25 @@ export function createSystemCollector({ return { read, - /** Start the deep scan, or attach to the one already running. Both callers - * get the SAME promise, so two concurrent refreshes can never race each - * other or double-write the snapshot (invariant 7). */ - refreshDeep() { + /** + * Start the deep scan, or attach to the one already running. Both callers + * get the SAME promise, so two concurrent refreshes can never race each + * other or double-write the snapshot (invariant 7). + * + * `includeProjectTrees` is a MEASUREMENT parameter, not a view filter: + * project working trees are walked or they are not, so changing it requires + * a rescan. Omitting it keeps the last value used — the chip that sets it + * reads its own state back off the last scan's payload. A caller that + * ATTACHES to a running scan gets that scan's parameter, not its own; the + * result says which was used (`consumers.includeProjectTrees` and + * `scan.includeProjectTrees`), so the answer is never mislabelled. + * + * @param {{ includeProjectTrees?: boolean }} [options] + */ + refreshDeep(options = {}) { + if (typeof options?.includeProjectTrees === 'boolean') { + includeProjectTrees = options.includeProjectTrees; + } if (inFlight) return inFlight; inFlight = runDeep().finally(() => { inFlight = null; }); return inFlight; diff --git a/src/lib/footprint/install.mjs b/src/lib/footprint/install.mjs index c5b5e53..56ea3d0 100644 --- a/src/lib/footprint/install.mjs +++ b/src/lib/footprint/install.mjs @@ -31,6 +31,7 @@ import { kbDir, present as brainPresent, installedVersion as brainVersion } from import { readJson } from '../settings.mjs'; import { walkTree, walkMeasurements, rootMeasurements, measured, unknown, sumMeasurements, statNode, + hasValue, } from './walk.mjs'; /** Install-method vocabulary. 'external' is the fail-closed value: the tool is @@ -61,6 +62,40 @@ const METHOD_RULES = Object.freeze([ export const MAX_NATIVE_ADDONS_PER_TOOL = 512; const NATIVE_EXT = '.node'; +/** The brain's install root is the PARENT of its KB dir. Measuring `kbDir()` + * alone under-reported this machine's brain by 85%: 1.9 GB of active KB inside + * a 13.2 GB cache root whose bulk is dated `kb.bak-*` copies the installer left + * behind on previous updates, plus the embedding models and its jsonl ledgers. + * + * Only the installer's own `…/ruvnet-brain/kb` layout is walked upward, and + * both segments must match. `RUVNET_BRAIN_KB` can relocate the KB anywhere — + * `/mnt/data/kb` would make the parent a directory the brain does not own, and + * billing a shared volume to the brain is a worse error than under-reporting + * it. An unrecognized layout stays measured at the KB dir, as before. */ +export function brainRoot() { + const kb = kbDir(); + const parent = path.dirname(kb); + const owned = path.basename(kb) === 'kb' && path.basename(parent) === 'ruvnet-brain'; + return owned ? parent : kb; +} + +/** Sub-rows for the brain's cache root, because the 85% that was invisible is + * not one number and must not become one: a user watching the figure jump from + * 1.9 GB to 13.2 GB is owed the reason. Order matters — `kb.bak-…` is tested + * before nothing, but `kb` is matched by exact equality so a backup can never + * fall into the active-KB bucket. */ +export const BRAIN_COMPONENTS = Object.freeze([ + { id: 'kb', label: 'Active knowledge base', match: (seg) => seg === 'kb' }, + { id: 'kb-backups', label: 'Superseded KB copies', match: (seg) => /^kb\.bak/i.test(seg) }, + { id: 'models', label: 'Embedding models', match: (seg) => seg === 'models' }, +]); + +/** Whatever no spec claimed. A breakdown whose parts do not add up to the whole + * is worse than no breakdown, so the remainder is always a row. */ +const COMPONENT_REMAINDER = Object.freeze({ + id: 'other', label: 'Ledgers and loose files', match: () => true, +}); + /** Attribute an install from a binary's REAL path. `globalRootDir`, when known, * wins over every manager rule: a package under npm's global node_modules is * npm-installed no matter which version manager owns the prefix. */ @@ -147,11 +182,13 @@ export function managedTools({ pkgRoot = null, globalRootDir = null } = {}) { kind: 'self', root: pkgRoot || npmRoot(KIT_PKG), }); tools.push({ - // Not an npm global: `npx ruvnet-brain` downloads a ~2 GB offline KB to a - // cache dir and wires a user-scope Claude Code plugin. Its bytes are the - // KB's, and its version comes from the plugin manifest, not npm. - id: 'ruvnet-brain', label: 'RuvNet Brain KB', pkg: null, bin: null, - kind: 'kb', root: kbDir(), + // Not an npm global: `npx ruvnet-brain` downloads an offline KB to a cache + // dir and wires a user-scope Claude Code plugin, so its version comes from + // the plugin manifest rather than npm. Its bytes are the WHOLE cache root's, + // not the active KB's — see brainRoot() for what that cost the figure — and + // the components say which part of the root they are. + id: 'ruvnet-brain', label: 'RuvNet Brain', pkg: null, bin: null, + kind: 'kb', root: brainRoot(), components: BRAIN_COMPONENTS, }); return tools; } @@ -195,10 +232,56 @@ function toolPresence(desc, fsImpl) { return dir.status === 'measured' && dir.kind === 'dir'; } +/** Bucket a tool tree's files by the top-level entry they live under, so a row + * can say WHY it is big instead of only how big. Buckets ride the SAME walk as + * the tree total — a second pass would double the I/O to answer one question, + * and two passes over a tree the installer rewrites nightly could disagree. + * Every component therefore inherits the walk's own provenance: a truncated or + * partly-degraded walk makes every component partial, and an unreadable root + * makes them unknown-with-reason rather than a set of tidy zeros. */ +function componentBuckets(specs, root) { + if (!specs?.length) return null; + const state = [...specs, COMPONENT_REMAINDER].map((spec) => ({ + spec, bytes: 0, files: 0, newestMtimeMs: null, names: new Set(), + })); + const identity = (s) => ({ + id: s.spec.id, + label: s.spec.label, + // One matched entry has a path worth printing; a family (five dated backups) + // or an empty bucket does not, and inventing one would name a directory that + // holds only part of the figure. + path: s.names.size === 1 ? path.join(root, [...s.names][0]) : null, + entries: s.names.size, + }); + return { + add({ file, bytes, mtimeMs }) { + const segment = path.relative(root, file).split(/[\\/]+/)[0] || ''; + const hit = state.find((s) => s.spec.match(segment)); + hit.bytes += bytes; + hit.files += 1; + hit.names.add(segment); + if (hit.newestMtimeMs === null || mtimeMs > hit.newestMtimeMs) hit.newestMtimeMs = mtimeMs; + }, + finalize({ asOf, partial }) { + return state.map((s) => ({ + ...identity(s), + bytes: measured(s.bytes, { asOf, partial }), + files: measured(s.files, { asOf, partial }), + newestMtimeMs: s.newestMtimeMs, + })); + }, + unmeasured(reason) { + return state.map((s) => ({ + ...identity(s), bytes: unknown(reason), files: unknown(reason), newestMtimeMs: null, + })); + }, + }; +} + /** * One HostInstallation. Shape: * { tool, label, package, present, version, installMethod, root, linkedFrom, - * rootReason, bytes, files, newestMtimeMs, nativeAddons[], + * rootReason, bytes, files, newestMtimeMs, components[], nativeAddons[], * nativeAddonCount, nativeAddonsTruncated, degraded[], complete } * * A tool that is genuinely not installed reports `measured(0)` bytes — zero is @@ -222,6 +305,7 @@ function collectTool(desc, ctx) { root: realRoot, linkedFrom, rootReason: null, + components: [], nativeAddons: [], nativeAddonCount: 0, nativeAddonsTruncated: false, @@ -263,10 +347,13 @@ function collectTool(desc, ctx) { const addons = []; let addonCount = 0; + const buckets = componentBuckets(desc.components, realRoot); const result = walk(realRoot, { ...limits, fsImpl, - onFile: ({ file, name, bytes, mtimeMs }) => { + onFile: (entry) => { + if (buckets) buckets.add(entry); + const { file, name, bytes, mtimeMs } = entry; if (!name.endsWith(NATIVE_EXT)) return; addonCount += 1; if (addons.length < maxNativeAddons) { @@ -281,6 +368,9 @@ function collectTool(desc, ctx) { bytes, files, newestMtimeMs: result.newestMtimeMs, + components: !buckets ? [] + : hasValue(bytes) ? buckets.finalize({ asOf, partial: bytes.partial }) + : buckets.unmeasured(bytes.reason), nativeAddons: addons, nativeAddonCount: addonCount, nativeAddonsTruncated: addonCount > addons.length, @@ -356,24 +446,100 @@ export function npxEnvNodes({ return { root, presence: 'present', reason: null, envs }; } +/** Playwright's browser cache has three platform locations and they are ONE + * cache, not three. Only two were listed, and the missing one was macOS's: + * `playwright install` writes to ~/Library/Caches/ms-playwright there, so a mac + * holding 1.86 GB of browser builds reported a measured zero — honest for the + * XDG path that genuinely does not exist, wrong for the question the row asks. + * + * All three are probed and they collapse into a single row, because two of them + * can be real at once: a cache migrated between layouts leaves both, and on + * macOS ~/.cache is sometimes a symlink into ~/Library/Caches, which is the same + * directory reachable by two names. Candidates are realpath-collapsed first — + * resolving a root the collector named itself, exactly as resolveRootPath does + * for a linked tool root — so an aliased target is measured once, and the + * survivors sum into one figure instead of near-identical rows that a total + * would add together. When none exists the platform-canonical path is the one + * named, so the measured zero says where it looked. */ +function playwrightCacheRoot({ env, platform, fsImpl }) { + const mac = path.join(home, 'Library', 'Caches', 'ms-playwright'); + const xdg = path.join(home, '.cache', 'ms-playwright'); + const win = path.join(env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'), 'ms-playwright'); + const candidates = platform === 'darwin' ? [mac, xdg, win] + : platform === 'win32' ? [win, xdg, mac] + : [xdg, mac, win]; + + const seen = new Set(); + const found = []; + for (const candidate of candidates) { + let real; + // realpath throws for a path that is not there; absence is what makes a + // candidate not a location, so it is the probe as well as the resolution. + try { real = fsImpl.realpathSync(candidate); } catch { continue; } + if (seen.has(real)) continue; + seen.add(real); + found.push(real); + } + return { + id: 'playwright', + label: 'Playwright browsers', + path: found[0] ?? candidates[0], + paths: found.length ? found : [candidates[0]], + }; +} + /** Install-adjacent shared caches. Deliberately their OWN rows rather than * smeared into a tool's tree: npx envs and browser binaries belong to no * single tool, and hiding them inside one would misattribute the bytes. The * brain KB is absent here on purpose — it is a managed tool with its own row. - * Windows browser caches live under LOCALAPPDATA; both candidates are listed - * and the platform's absent one reads as a measured zero. */ -export function sharedCacheRoots({ env = process.env } = {}) { - const localAppData = env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'); + * + * `paths` is the measured set and `path` is the one to print; they differ only + * where a cache legitimately lives in more than one place (see + * playwrightCacheRoot). */ +export function sharedCacheRoots({ + env = process.env, platform = process.platform, fsImpl = fs, +} = {}) { + const single = (id, label, dir) => ({ id, label, path: dir, paths: [dir] }); return [ - { id: 'npx-envs', label: 'npx cache envs', path: npxCacheDir() }, - { id: 'claude-plugins', label: 'Claude Code plugins', path: path.join(claudeDir(), 'plugins') }, - { id: 'codex-plugins', label: 'Codex plugin cache', path: codexPluginCacheDir() }, - { id: 'playwright', label: 'Playwright browsers', path: path.join(home, '.cache', 'ms-playwright') }, - { id: 'playwright-win', label: 'Playwright browsers', path: path.join(localAppData, 'ms-playwright') }, - { id: 'puppeteer', label: 'Puppeteer browsers', path: path.join(home, '.cache', 'puppeteer') }, + single('npx-envs', 'npx cache envs', npxCacheDir()), + single('claude-plugins', 'Claude Code plugins', path.join(claudeDir(), 'plugins')), + single('codex-plugins', 'Codex plugin cache', codexPluginCacheDir()), + playwrightCacheRoot({ env, platform, fsImpl }), + single('puppeteer', 'Puppeteer browsers', path.join(home, '.cache', 'puppeteer')), ]; } +/** One shared-cache row's figures. The single-location case is the exact + * rootMeasurements result, untouched: an errno on a lone root must reach the UI + * as that errno, and folding it through a sum would flatten it to "every input + * unmeasured". Only a genuinely multi-location cache merges, and it merges + * conservatively — present beats degraded beats absent, and an unreadable + * location makes the sum partial rather than silently dropping its bytes. */ +function measureCacheRoot(cache, { walk, limits, fsImpl, asOf }) { + const locations = cache.paths?.length ? cache.paths : [cache.path]; + const parts = locations.map((dir) => { + const result = walk(dir, { ...limits, fsImpl }); + return { result, ...rootMeasurements(result, { asOf }) }; + }); + if (parts.length === 1) { + const [only] = parts; + return { + presence: only.presence, bytes: only.bytes, files: only.files, + newestMtimeMs: only.result.newestMtimeMs, complete: only.result.complete, + }; + } + const presence = parts.some((p) => p.presence === 'present') ? 'present' + : parts.some((p) => p.presence === 'degraded') ? 'degraded' : 'absent'; + const mtimes = parts.map((p) => p.result.newestMtimeMs).filter((m) => m !== null); + return { + presence, + bytes: sumMeasurements(parts.map((p) => p.bytes), { asOf }), + files: sumMeasurements(parts.map((p) => p.files), { asOf }), + newestMtimeMs: mtimes.length ? Math.max(...mtimes) : null, + complete: parts.every((p) => p.result.complete !== false), + }; +} + /** The section's denominator: "the install is X GB" is meaningless without an * "of Y free" beside it. statfs is a Node builtin (>=18.15) — no dependency, * and it works on Windows too. Failure reports unknown, never 0. */ @@ -425,13 +591,8 @@ export function collectInstall({ const sharedCaches = []; let npxEnvs = { root: npxCacheDir(), presence: 'unknown', reason: 'not collected', envs: [] }; if (includeCaches) { - for (const cache of sharedCacheRoots()) { - const result = walk(cache.path, { ...limits, fsImpl }); - const { presence, bytes, files } = rootMeasurements(result, { asOf }); - sharedCaches.push({ - ...cache, presence, bytes, files, newestMtimeMs: result.newestMtimeMs, - complete: result.complete, - }); + for (const cache of sharedCacheRoots({ fsImpl })) { + sharedCaches.push({ ...cache, ...measureCacheRoot(cache, { walk, limits, fsImpl, asOf }) }); } npxEnvs = npxEnvNodes({ walk, limits, asOf, fsImpl }); } diff --git a/src/lib/footprint/project-sources.mjs b/src/lib/footprint/project-sources.mjs new file mode 100644 index 0000000..6d131cc --- /dev/null +++ b/src/lib/footprint/project-sources.mjs @@ -0,0 +1,448 @@ +// Project sources — every project this machine has EVER had a Claude, Codex or +// OpenCode session with, de-duplicated across hosts by resolved real path. +// +// This deliberately does NOT reuse `discoverRuvfloProjects()`. That function +// answers a different question — "which projects carry ruflo learning state" — +// by requiring a `.claude-flow/neural/` directory and by reading only the 150 +// most-recently-modified transcripts per host. Both narrowings are correct +// there and wrong here: on this machine they collapse 48 projects to 5. The +// Intelligence panel still depends on that meaning, so it keeps it, and the +// System area gets its own source with its own contract. +// +// Two figures come out of this and they are NOT the same number: +// everSeen every distinct project any host ever recorded a session in, +// including the ones that have since been deleted or moved — the +// deletions are the point, so they are never dropped; +// onDisk the subset that still resolves to a directory, i.e. the only +// ones a byte/LOC measurement can be taken of at all. +// +// Content boundary. This is DISCOVERY, invariant 9's candidate-path source, not +// a measurement: it reads ONE field out of a transcript — the session's `cwd` — +// and nothing else. The same read `native-transcript-discovery.mjs` already +// performs for Observability at the same trust boundary. No message, prompt or +// tool payload is parsed, retained or emitted; every figure the System area +// renders is measured downstream by walk.mjs-backed collectors from the paths +// this module returns. +// +// Cost. The corpus here is ~3,200 transcripts. Each file is opened once and +// only its HEAD is read (HEAD_BYTES, JSON-parsed up to HEAD_MAX_LINES non-blank +// lines) — a session's cwd is recorded in its opening records or not at all, so +// reading further would cost the whole corpus to learn nothing. A file that +// cannot be read or parsed is counted and skipped; one bad transcript never +// aborts the walk (invariant 6). +import fs from 'node:fs'; +import path from 'node:path'; +import { claudeDir, codexDir } from '../paths.mjs'; +import { resolveProjectLabel } from '../live/index.mjs'; +import { withDb } from '../sqlite.mjs'; +import { defaultOpencodeDbPath } from '../usage-opencode.mjs'; +import { presenceOf, statNode, UNKNOWN, walkTree } from './walk.mjs'; + +/** Hosts in the order every payload lists them. */ +export const PROJECT_SOURCE_HOSTS = Object.freeze(['claude', 'codex', 'opencode']); + +/** How much of a transcript is read looking for its cwd, and how many of its + * leading non-blank lines are JSON-parsed. Both are budgets, not guesses: a + * head that carries no cwd is reported as such rather than searched further. */ +export const HEAD_BYTES = 256 * 1024; +export const HEAD_MAX_LINES = 40; + +/** `~/.claude/projects//.jsonl` is 2 deep; codex rollouts are + * `sessions/YYYY/MM/DD/.jsonl`, 4 deep. 8 leaves room for either root + * gaining a level without letting an unexpected tree run away. */ +const TRANSCRIPT_MAX_DEPTH = 8; + +/** lstat budget for one encoded-directory decode. The decode is a bounded + * search (below), so it needs a ceiling of its own; 512 covers a deep path + * with several ambiguous segments. */ +const DECODE_STAT_BUDGET = 512; + +/** The one-line statement of what was counted and how, so no surface can render + * these numbers without being able to say where they came from. */ +export const PROJECT_SOURCE_METHOD = + 'every cwd named by a Claude or Codex transcript head, plus every OpenCode session ' + + 'directory, de-duplicated by resolved real path — not only projects with ruflo state'; + +// ── transcript heads ────────────────────────────────────────────────────────── + +/** Leading non-blank lines of a file's head. `null` means the file could not be + * read at all — distinct from an empty file, which is a real, readable zero + * lines. The buffer is parsed and discarded; nothing from it is retained. */ +function readHeadLines(file, { fsImpl = fs, headBytes = HEAD_BYTES, maxLines = HEAD_MAX_LINES } = {}) { + let fd; + try { fd = fsImpl.openSync(file, 'r'); } catch { return null; } + try { + const size = Math.min(fsImpl.fstatSync(fd).size, headBytes); + if (size === 0) return []; + const buffer = Buffer.allocUnsafe(size); + const read = fsImpl.readSync(fd, buffer, 0, size, 0); + const lines = []; + for (const line of buffer.toString('utf8', 0, read).split('\n')) { + if (!line.trim()) continue; + lines.push(line); + if (lines.length >= maxLines) break; + } + return lines; + } catch { return null; } + finally { try { fsImpl.closeSync(fd); } catch { /* fd already gone */ } } +} + +/** + * The first session cwd a transcript head declares, or null. + * + * Claude writes a flat `record.cwd` on its own records; Codex writes + * `record.payload.cwd` on the `session_meta` / `turn_context` records that open + * a rollout. A line that does not parse is skipped — a truncated final line in + * a head window is expected, not a failure. + */ +export function firstCwd(lines, host) { + for (const line of lines ?? []) { + let record; + try { record = JSON.parse(line); } catch { continue; } + if (!record || typeof record !== 'object') continue; + const cwd = host === 'claude' + ? record.cwd + : (['session_meta', 'turn_context'].includes(record.type) ? record.payload?.cwd : null); + if (typeof cwd === 'string' && cwd) return cwd; + } + return null; +} + +// ── the encoded Claude project directory ────────────────────────────────────── + +/** + * Best-effort decode of a `~/.claude/projects/` directory name back to the path + * it encodes, used ONLY as a fallback for a project directory whose transcripts + * carry no cwd record. + * + * The encoding is LOSSY: `/`, `.` and a literal `-` all become `-`, so + * `-Users-me-ai-agentic-kit` is equally readable as `/Users/me/ai/agentic/kit` + * and `/Users/me/ai/agentic-kit`. There is therefore no safe pure-string + * decode, and this does not attempt one — it walks the candidate segments + * against the real filesystem and returns a path only when the filesystem + * confirms it. The consequence is stated rather than hidden: a project whose + * directory is GONE cannot be recovered this way, so those groups are counted + * as `unresolved` and make `everSeen` a lower bound instead of being guessed at. + * + * @param {string} name the encoded directory name + * @param {{ fsImpl?: typeof fs, budget?: number }} [options] + * @returns {string|null} + */ +export function decodeClaudeProjectDir(name, { fsImpl = fs, budget = DECODE_STAT_BUDGET } = {}) { + const tokens = String(name ?? '').split('-'); + // A leading empty token is the leading `/`. Anything else (a Windows drive + // prefix, a relative name) is not decodable here and says so by returning null. + if (tokens[0] !== '' || tokens.length < 2) return null; + const rest = tokens.slice(1); + let stats = 0; + const isDir = (target) => { + stats += 1; + const node = statNode(target, { fsImpl }); + return node.status !== UNKNOWN && node.kind === 'dir'; + }; + + const advance = (base, index) => { + if (stats > budget) return null; + if (index >= rest.length) return base; + let joined = ''; + for (let end = index; end < rest.length; end++) { + // The separator swallowed by the encoding was either a literal `-` or a + // `.` (a dotted directory such as `.claude`); both are tried, shortest + // segment first, and the filesystem decides. + for (const separator of end === index ? [''] : ['-', '.']) { + const candidate = end === index ? rest[end] : `${joined}${separator}${rest[end]}`; + const next = path.join(base, candidate); + if (!isDir(next)) continue; + const resolved = advance(next, end + 1); + if (resolved) return resolved; + } + joined = joined === '' ? rest[end] : `${joined}-${rest[end]}`; + } + return null; + }; + return advance(path.sep, 0); +} + +// ── per-host scans ──────────────────────────────────────────────────────────── + +/** A walk root's health in the same vocabulary the storage collector uses: a + * root that does not exist is an ABSENCE (that host was never used here), not + * a failed measurement. */ +function rootStatus(walkResult) { + const presence = presenceOf(walkResult); + return presence === 'present' ? 'ok' : presence; +} + +/** + * Every cwd named by the transcripts under `root`, plus the counts a liner note + * needs to state what was and was not recoverable. + * + * @param {string} root + * @param {'claude'|'codex'} host + * @param {{ walk?: Function, fsImpl?: typeof fs, headBytes?: number, maxLines?: number, + * decodeDir?: ((name: string) => string|null)|null, readHead?: Function }} [options] + * `decodeDir` enables the encoded-directory fallback (Claude only — Codex + * rollout directories are dated, not project-scoped, so there is nothing to + * decode). + */ +export function scanTranscriptCwds(root, host, { + walk = walkTree, fsImpl = fs, headBytes = HEAD_BYTES, maxLines = HEAD_MAX_LINES, + decodeDir = null, readHead = readHeadLines, +} = {}) { + const files = []; + const result = walk(root, { + maxDepth: TRANSCRIPT_MAX_DEPTH, + fsImpl, + acceptFile: (name) => name.endsWith('.jsonl'), + onFile: ({ file, mtimeMs }) => { files.push({ file, mtimeMs }); }, + }); + const status = rootStatus(result); + const base = { + host, + root, + status, + reason: status === 'ok' ? null : (result.reason ?? null), + files: 0, + withCwd: 0, + withoutCwd: 0, + empty: 0, + unreadable: 0, + unresolved: 0, + recoveredFromDirName: 0, + sightings: [], + truncated: Boolean(result.truncated), + truncatedBy: result.truncatedBy ?? null, + degraded: result.degraded ?? [], + complete: result.complete !== false, + }; + if (status !== 'ok') return { ...base, complete: status === 'absent' }; + + const sightings = []; + // Grouped by the encoded project directory so the fallback can be applied to + // a directory as a whole: one transcript without a cwd is irrelevant while a + // sibling has one, and only a group with NO cwd anywhere is a lost project. + const groups = new Map(); + let withCwd = 0; + let withoutCwd = 0; + let empty = 0; + let unreadable = 0; + + for (const { file, mtimeMs } of files) { + let group = null; + if (decodeDir) { + const key = path.relative(root, file).split(path.sep)[0]; + group = groups.get(key); + if (!group) { group = { key, withCwd: false, newestMtimeMs: null }; groups.set(key, group); } + if (Number.isFinite(mtimeMs) && (group.newestMtimeMs === null || mtimeMs > group.newestMtimeMs)) { + group.newestMtimeMs = mtimeMs; + } + } + const lines = readHead(file, { fsImpl, headBytes, maxLines }); + if (lines === null) { unreadable += 1; continue; } + if (lines.length === 0) { empty += 1; continue; } + const cwd = firstCwd(lines, host); + if (!cwd) { withoutCwd += 1; continue; } + withCwd += 1; + sightings.push({ cwd, mtimeMs, origin: 'cwd' }); + if (group) group.withCwd = true; + } + + let unresolved = 0; + let recoveredFromDirName = 0; + for (const group of groups.values()) { + if (group.withCwd) continue; + const decoded = decodeDir(group.key); + if (decoded) { + recoveredFromDirName += 1; + sightings.push({ cwd: decoded, mtimeMs: group.newestMtimeMs, origin: 'encoded-dir' }); + } else { + unresolved += 1; + } + } + + return { + ...base, + files: files.length, + withCwd, + withoutCwd, + empty, + unreadable, + unresolved, + recoveredFromDirName, + sightings, + // Transcripts we could not read, and project directories whose path could + // not be recovered, both mean the project list is a floor. + complete: result.complete !== false && unreadable === 0 && unresolved === 0, + }; +} + +/** + * OpenCode session directories. Unlike the JSONL hosts, OpenCode keeps sessions + * in a single SQLite store whose `session` table carries the session's own + * `directory` — an absolute path — so its projects ARE recoverable and are read + * from there rather than guessed at. The store is opened READ-ONLY and only + * that one column is selected. + * + * An absent store means OpenCode was never used on this machine (a real zero); + * a store that will not open, or an older schema without `directory`, is + * reported degraded with its reason rather than silently contributing nothing. + */ +export function scanOpencodeDirectories({ dbFile = defaultOpencodeDbPath(), withDb: withDbImpl = withDb } = {}) { + const base = { host: 'opencode', root: dbFile, status: 'ok', reason: null, sessions: 0, sightings: [], complete: true }; + const result = withDbImpl(dbFile, (db) => db.prepare( + 'SELECT directory, COUNT(*) AS sessions, MAX(COALESCE(time_updated, time_created)) AS lastMs' + + " FROM session WHERE directory IS NOT NULL AND directory <> '' GROUP BY directory", + ).all()); + if (!result.ok) { + const absent = result.error?.kind === 'absent'; + return { + ...base, + status: absent ? 'absent' : 'degraded', + reason: absent ? null : (result.error?.message ?? 'store unreadable'), + complete: absent, + }; + } + const sightings = []; + let sessions = 0; + for (const row of result.value ?? []) { + const count = Number(row?.sessions) || 0; + sessions += count; + const lastMs = Number(row?.lastMs); + sightings.push({ + cwd: row?.directory, + mtimeMs: Number.isFinite(lastMs) ? lastMs : null, + origin: 'cwd', + weight: count, + }); + } + return { ...base, sessions, sightings }; +} + +// ── assembly ────────────────────────────────────────────────────────────────── + +/** Canonicalize for de-dup so one project touched by two hosts, or reached + * through a symlink, is ONE project. Falls back to path.resolve when the target + * cannot be realpath'd — a deleted project has no real path, and it must still + * be counted. */ +function resolvePath(candidate, fsImpl) { + try { return (fsImpl.realpathSync.native ?? fsImpl.realpathSync)(candidate); } + catch { return path.resolve(candidate); } +} + +/** Observability's label for the same path, so a project reads identically in + * both areas; the bare directory name when that cannot be derived. */ +function labelFor(resolveLabel, resolved) { + try { + const label = resolveLabel(resolved); + if (typeof label === 'string' && label && label !== 'unknown') return label; + } catch { /* fall through to the basename */ } + return path.basename(resolved) || resolved; +} + +/** A `.git` DIRECTORY is a repository; a `.git` FILE is a linked worktree, which + * is equally a repository. Checking only for a directory would undercount every + * worktree on the machine. */ +function gitPresence(projectPath, fsImpl) { + const node = statNode(path.join(projectPath, '.git'), { fsImpl }); + return node.status !== UNKNOWN && (node.kind === 'dir' || node.kind === 'file'); +} + +/** + * Every project any host has ever recorded a session in. + * + * @param {{ claudeRoot?: string, codexRoot?: string, opencodeDbFile?: string, + * walk?: Function, fsImpl?: typeof fs, now?: () => number, + * resolveLabel?: Function, scanTranscripts?: Function, + * scanOpencode?: Function, headBytes?: number, maxLines?: number, + * decodeEncodedDirs?: boolean }} [options] + * @returns {{ + * asOf: number, + * projects: Array<{ path: string, label: string, hosts: string[], origins: string[], + * exists: boolean, isGitRepo: boolean, lastSeenMs: number|null, + * sessions: number }>, + * everSeen: number, onDisk: number, gitRepos: number, unresolved: number, + * complete: boolean, method: string, + * sources: Record<'claude'|'codex'|'opencode', object>, + * }} `everSeen` counts projects INCLUDING vanished ones; `onDisk` counts the + * measurable subset. `complete: false` means at least one transcript or + * project directory could not be resolved, so both counts are lower bounds. + */ +export function discoverProjectSources({ + claudeRoot = path.join(claudeDir(), 'projects'), + codexRoot = path.join(codexDir(), 'sessions'), + opencodeDbFile = defaultOpencodeDbPath(), + walk = walkTree, + fsImpl = fs, + now = Date.now, + resolveLabel = resolveProjectLabel, + scanTranscripts = scanTranscriptCwds, + scanOpencode = scanOpencodeDirectories, + headBytes = HEAD_BYTES, + maxLines = HEAD_MAX_LINES, + decodeEncodedDirs = true, +} = {}) { + const asOf = now(); + const transcriptOpts = { walk, fsImpl, headBytes, maxLines }; + const sources = { + claude: scanTranscripts(claudeRoot, 'claude', { + ...transcriptOpts, + decodeDir: decodeEncodedDirs ? (name) => decodeClaudeProjectDir(name, { fsImpl }) : null, + }), + codex: scanTranscripts(codexRoot, 'codex', transcriptOpts), + opencode: scanOpencode({ dbFile: opencodeDbFile }), + }; + + const byPath = new Map(); + for (const host of PROJECT_SOURCE_HOSTS) { + for (const sighting of sources[host]?.sightings ?? []) { + const cwd = sighting?.cwd; + if (typeof cwd !== 'string' || !cwd || !path.isAbsolute(cwd)) continue; + const resolved = resolvePath(cwd, fsImpl); + let row = byPath.get(resolved); + if (!row) { + row = { path: resolved, hosts: new Set(), origins: new Set(), sessions: 0, lastSeenMs: null }; + byPath.set(resolved, row); + } + row.hosts.add(host); + row.origins.add(sighting.origin ?? 'cwd'); + row.sessions += Number.isFinite(sighting.weight) ? sighting.weight : 1; + const at = sighting.mtimeMs; + if (Number.isFinite(at) && (row.lastSeenMs === null || at > row.lastSeenMs)) row.lastSeenMs = at; + } + } + + const projects = [...byPath.values()].map((row) => { + const node = statNode(row.path, { fsImpl }); + const exists = node.status !== UNKNOWN && node.kind === 'dir'; + return { + path: row.path, + label: labelFor(resolveLabel, row.path), + hosts: PROJECT_SOURCE_HOSTS.filter((host) => row.hosts.has(host)), + origins: [...row.origins].sort(), + exists, + // A path that is gone is not a repository and is not "not a repository" + // either — but `false` is the only honest reading available, and `exists` + // sits next to it so no consumer can mistake the two. + isGitRepo: exists && gitPresence(row.path, fsImpl), + lastSeenMs: row.lastSeenMs, + sessions: row.sessions, + }; + }); + // Most-recently-seen first; a project with no usable timestamp sorts last but + // is never dropped, and the path tiebreak keeps the order stable. + projects.sort((a, b) => (b.lastSeenMs ?? -1) - (a.lastSeenMs ?? -1) || a.path.localeCompare(b.path)); + + const unresolved = PROJECT_SOURCE_HOSTS + .reduce((total, host) => total + (sources[host]?.unresolved ?? 0), 0); + return { + asOf, + projects, + everSeen: projects.length, + onDisk: projects.filter((project) => project.exists).length, + gitRepos: projects.filter((project) => project.isGitRepo).length, + unresolved, + complete: PROJECT_SOURCE_HOSTS.every((host) => sources[host]?.complete !== false), + method: PROJECT_SOURCE_METHOD, + sources, + }; +} diff --git a/src/lib/footprint/projects.mjs b/src/lib/footprint/projects.mjs index b83fc98..708eb1c 100644 --- a/src/lib/footprint/projects.mjs +++ b/src/lib/footprint/projects.mjs @@ -1,84 +1,73 @@ -// Project footprints — one row per project in the shared discovery catalog: +// Project footprints — one row per project this machine has had a session with: // approximate LOC by language, working-tree bytes, `.git` bytes, `node_modules` // bytes, last activity, and the origin remote's web page when one exists. // +// TWO POPULATIONS, deliberately not the same number. The TABLE covers projects +// that still exist on disk, because those are the only ones a byte or LOC figure +// can be taken of at all. The KPI counts, carried alongside it, cover every +// project ever seen — including the ones that have since been deleted, which is +// exactly the fact a "how many projects has this machine touched" question is +// asking about. `project-sources.mjs` produces both; a consumer that renders +// only one of them must say which (`method` travels with the payload for that). +// // The three byte figures stay SEPARATE on purpose (ADR-0025): `node_modules` // dominates and `.git` distorts, so folding either into the tree would let // reinstallable overhead masquerade as "your project got big". // +// WHAT A PROJECT IS MADE OF comes from the stack registry (`stack-detect.mjs`), +// not from a map kept here. Two kinds of fact come back and they are kept apart: +// +// languages carry LINES — an extension is what a line belongs to, so these are +// what the stacked bar renders; +// stack (frameworks / SDKs / tools) carries PRESENCE ONLY and is never +// given a line count. React does not own lines, the .tsx files do, +// and putting both on one proportional bar would count the same bytes +// twice. The field is structurally absent from the payload, so no +// surface can make that mistake by accident. +// // LOC is APPROXIMATE by invariant 11 and says so in the payload: it is an // extension-bucketed newline count with a stated exclusion list, produced by the -// kit's own bounded walker (zero runtime dependencies — no cloc, no tokei). Files -// are scanned through a fixed 64 KB buffer that is counted and discarded; no file -// content is ever retained or emitted. +// kit's own bounded walker (zero runtime dependencies — no cloc, no tokei). An +// extension the registry does not map is NEVER counted — the top unmapped +// extensions on a real machine are .jsonl/.png/.jar/.dll, i.e. data and binaries, +// and counting them would corrupt the figure. They are carried through BY NAME +// instead, as the unrecognized tail, so "Other" renders as a named to-do rather +// than a shrug. // // Discovery supplies PATHS ONLY (invariant 9). Every figure below is measured here. import fs from 'node:fs'; import path from 'node:path'; import { parseRepoSlug } from '../admin-collect.mjs'; -import { discoverRuvfloProjects } from '../dashboard/project-discovery.mjs'; -import { walkTree, rootMeasurements, measured, unknown, sumMeasurements } from './walk.mjs'; +import { discoverProjectSources } from './project-sources.mjs'; +import { detectStack, STACK_EXCLUSIONS } from './stack-detect.mjs'; +import { STACK_REGISTRY_VERSION } from './stack-registry.mjs'; +import { + walkTree, rootMeasurements, measured, statNode, UNKNOWN, unknown, sumMeasurements, +} from './walk.mjs'; /** Directories that are never code and never the user's work. Excluded from the * tree walk, from LOC, and from the nested-`node_modules` search alike, so the * three byte figures partition the project rather than overlapping it. */ const OVERHEAD_DIRS = new Set(['.git', 'node_modules']); -/** Vendored / generated / virtual-env trees. Excluded from LOC only: they are - * real bytes on disk (so they stay in treeBytes) but they are not lines the user - * wrote, and counting them would make the LOC figure meaningless. */ -const LOC_EXCLUDED_DIRS = new Set([ - 'node_modules', '.git', 'vendor', 'third_party', 'thirdparty', 'bower_components', - 'dist', 'build', 'out', 'target', '.next', '.nuxt', '.svelte-kit', 'coverage', - '.venv', 'venv', '__pycache__', '.tox', '.mypy_cache', '.pytest_cache', - '.gradle', '.idea', '.vscode', 'Pods', '.terraform', '.cache', '.turbo', -]); - -/** Machine-generated manifests: text, enormous, and nobody's line count. */ -const LOC_EXCLUDED_FILES = new Set([ - 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'npm-shrinkwrap.json', - 'Cargo.lock', 'poetry.lock', 'Gemfile.lock', 'composer.lock', 'go.sum', 'flake.lock', -]); - -/** Extension → language bucket. An extension absent from this map is NOT counted - * — an unknown extension may be a binary, and guessing would inflate the total. */ -const LANGUAGES = new Map(Object.entries({ - '.js': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript', '.jsx': 'javascript', - '.ts': 'typescript', '.tsx': 'typescript', '.mts': 'typescript', '.cts': 'typescript', - '.rs': 'rust', '.go': 'go', '.py': 'python', '.rb': 'ruby', '.php': 'php', - '.java': 'java', '.kt': 'kotlin', '.kts': 'kotlin', '.scala': 'scala', '.groovy': 'groovy', - '.cs': 'csharp', '.fs': 'fsharp', '.swift': 'swift', '.m': 'objective-c', '.mm': 'objective-c', - '.c': 'c', '.h': 'c', '.cc': 'cpp', '.cpp': 'cpp', '.cxx': 'cpp', '.hpp': 'cpp', '.hh': 'cpp', - '.ex': 'elixir', '.exs': 'elixir', '.erl': 'erlang', '.hs': 'haskell', '.clj': 'clojure', - '.lua': 'lua', '.dart': 'dart', '.zig': 'zig', '.nim': 'nim', '.pl': 'perl', '.r': 'r', - '.sh': 'shell', '.bash': 'shell', '.zsh': 'shell', '.fish': 'shell', '.ps1': 'powershell', - '.sql': 'sql', '.vue': 'vue', '.svelte': 'svelte', - '.html': 'html', '.htm': 'html', '.css': 'css', '.scss': 'css', '.sass': 'css', '.less': 'css', - '.md': 'markdown', '.mdx': 'markdown', - '.json': 'config', '.yml': 'config', '.yaml': 'config', '.toml': 'config', - '.ini': 'config', '.xml': 'config', '.proto': 'config', '.tf': 'config', -})); - -/** Source trees nest far shallower than dependency trees; 12 covers a monorepo - * package's deepest source dir without letting a pathological tree run away. */ -const LOC_MAX_DEPTH = 12; -/** Files above this are minified bundles, fixtures, or data dumps far more often - * than they are hand-written source. Skipped and reported, not counted. */ -const LOC_MAX_FILE_BYTES = 2 * 1024 * 1024; -const READ_CHUNK = 64 * 1024; /** Depth at which a workspace's nested `node_modules` still gets attributed. */ const NODE_MODULES_MAX_DEPTH = 6; /** The exclusion list every consumer must be able to state alongside the figure * (invariant 11). Deliberate exclusions are why LOC is approximate; they are NOT - * a failed measurement, so they never mark the count partial. */ -export const LOC_EXCLUSIONS = Object.freeze([ - ...[...LOC_EXCLUDED_DIRS].sort().map((dir) => `${dir}/`), - ...[...LOC_EXCLUDED_FILES].sort(), - 'files without a recognized source extension', - 'files containing NUL bytes (binary)', - `files larger than ${LOC_MAX_FILE_BYTES} bytes`, -]); + * a failed measurement, so they never mark the count partial. + * + * Retained under its original name for existing consumers: the list is now + * STATED BY THE REGISTRY-BACKED DETECTOR rather than assembled here, so the + * exclusions a project row prints and the ones the scan actually applied cannot + * drift apart. */ +export const LOC_EXCLUSIONS = STACK_EXCLUSIONS; + +/** Aggregate tail caps, matching stack-detect's per-project ones: a machine-wide + * to-do list longer than this is not read, and the totals beside it state what + * the slice left out. */ +const SECTION_TAIL_EXTENSIONS = 40; +const SECTION_TAIL_DEPENDENCIES = 50; // ── git remote ──────────────────────────────────────────────────────────────── @@ -169,6 +158,16 @@ export function describeRemote(rawUrl, name = 'origin') { export function projectRemote(projectPath, { fsImpl = fs } = {}) { const configFile = path.join(projectPath, '.git', 'config'); let source; + // Stat before read: a cloud provider's evicted placeholder (real size, zero + // allocated blocks) stats instantly but blocks forever on open, waiting for a + // provider that may be signed out. `unknown` with a stated reason is the + // honest answer — inventing `local-only` would claim this repo has no remote. + try { + const st = fsImpl.lstatSync(configFile); + if (st.blocks === 0 && st.size > 0) { + return { status: 'unknown', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: 'cloud placeholder (not materialized)' }; + } + } catch { /* the read below reports the errno; one stat failure decides nothing */ } try { source = fsImpl.readFileSync(configFile, 'utf8'); } catch (error) { if (error.code === 'ENOENT' || error.code === 'ENOTDIR') { return { status: 'local-only', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: null }; @@ -182,94 +181,142 @@ export function projectRemote(projectPath, { fsImpl = fs } = {}) { return { ...describeRemote(remote.url, remote.name), reason: null }; } -// ── lines of code ───────────────────────────────────────────────────────────── +// ── lines of code, and what the lines are written in ────────────────────────── +// +// Both sections below are PROJECTIONS of one `detectStack()` pass. The pass is +// run once per project and split here rather than measured twice: a second walk +// would double the I/O of the most expensive part of the scan to re-derive facts +// the first walk already held. -/** Count newlines in one file through a fixed buffer. Returns null when the file - * is binary (a NUL byte in the first chunk) or unreadable — never 0, which would - * claim an empty file. Each chunk is counted and immediately overwritten; no file - * content is retained past this function or emitted anywhere. */ -function countFileLines(file, size, fsImpl = fs) { - let fd; - try { fd = fsImpl.openSync(file, 'r'); } catch { return null; } - try { - const buffer = Buffer.allocUnsafe(READ_CHUNK); - let lines = 0; - let read = 0; - let lastByte = 0; - let offset = 0; - let first = true; - while ((read = fsImpl.readSync(fd, buffer, 0, READ_CHUNK, offset)) > 0) { - // One NUL in the first chunk is the cheap, conventional binary test. - if (first) { const nul = buffer.indexOf(0); if (nul >= 0 && nul < read) return null; } - first = false; - for (let i = 0; i < read; i++) if (buffer[i] === 0x0a) lines++; - lastByte = buffer[read - 1]; - offset += read; - } - // A final line with no trailing newline still counts as a line. - if (size > 0 && lastByte !== 0x0a) lines++; - return lines; - } catch { return null; } - finally { try { fsImpl.closeSync(fd); } catch { /* fd already gone */ } } -} +/** Was there a measurement at all? `detectStack` reports an unwalkable root with + * an unknown `totalLines` and empty lists — and an empty list of languages is + * indistinguishable from "this project has none" unless the caller checks. */ +const stackMeasured = (detected) => Boolean(detected?.totalLines) + && detected.totalLines.status !== UNKNOWN; /** - * Approximate lines of code under `root`, bucketed by language, via the shared - * bounded walker — so LOC inherits the same never-follow-symlinks rule, entry - * caps, and degrade-this-node-only failure mode as every other figure here. + * The LOC projection: lines and only lines. * - * `total` is `partial` only when a cap fired or a subtree was unreadable. The - * exclusion list is a deliberate scope, not a failure, and never marks it partial - * — which is precisely why the figure ships with `approximate` and `exclusions` - * attached, so no consumer can render it as authoritative (invariant 11). + * `byLanguage` (bucket id → lines) is kept for the surfaces that already read it; + * `languages` is the ranked, registry-described form the stacked bar renders, + * each row carrying the palette SLOT the registry assigned rather than a colour. + * Both come from the same detection, so they cannot disagree. * - * @param {string} root - * @param {{ walk?: Function, limits?: object, maxDepth?: number, asOf?: number|null, - * fsImpl?: typeof fs }} [options] - * @returns {object} LocCount + * `total` is `partial` only when a cap fired or a subtree was unreadable — the + * exclusion list is a deliberate scope, not a failure, which is precisely why the + * figure ships with `approximate` and `exclusions` attached (invariant 11). */ -export function countLines(root, { - walk = walkTree, limits = {}, maxDepth = LOC_MAX_DEPTH, asOf = null, fsImpl = fs, -} = {}) { +function locFromStack(detected) { + const base = { + approximate: true, + exclusions: [...STACK_EXCLUSIONS], + registryVersion: detected?.registryVersion ?? STACK_REGISTRY_VERSION, + }; + if (!stackMeasured(detected)) { + return { + ...base, + total: detected?.totalLines ?? unknown('not measured'), + byLanguage: null, + languages: null, + files: null, + skipped: 0, + complete: false, + degraded: detected?.degraded ?? [], + }; + } + const languages = detected.languages ?? []; const byLanguage = {}; - let total = 0; - let files = 0; - let skipped = 0; - const languageOf = (name) => LANGUAGES.get(path.extname(name).toLowerCase()); - - const result = walk(root, { - maxDepth, ...limits, fsImpl, - skipDir: (dir, name) => LOC_EXCLUDED_DIRS.has(name), - acceptFile: (name) => !LOC_EXCLUDED_FILES.has(name) && Boolean(languageOf(name)), - onFile: ({ file, name, bytes }) => { - if (bytes > LOC_MAX_FILE_BYTES) { skipped++; return; } - const lines = countFileLines(file, bytes, fsImpl); - if (lines === null) { skipped++; return; } - const language = languageOf(name); - byLanguage[language] = (byLanguage[language] ?? 0) + lines; - total += lines; - files++; - }, - }); + for (const row of languages) byLanguage[row.id] = row.lines; + return { + ...base, + total: detected.totalLines, + byLanguage, + // Ranked by lines already; copied so a consumer cannot mutate the detection. + languages: languages.map((row) => ({ ...row })), + files: detected.files, + skipped: detected.skipped ?? 0, + // The WALK's completeness, read off the measurement the walk stamped. A + // manifest that would not parse is a stack fact, not a line-count fact: + // `detected.complete` folds both in, so it is the wrong signal here. + complete: detected.totalLines?.partial !== true, + degraded: detected.degraded ?? [], + }; +} - const base = { approximate: true, exclusions: [...LOC_EXCLUSIONS] }; - if (result.status === 'unknown') { +/** + * The stack projection: PRESENCE, never lines. + * + * `items` are frameworks / SDKs / tools, each with the `kind` it was registered + * under and the `via` that matched it. None of them carries a line count and none + * ever may — see this file's header. + * + * `unrecognized` is the tail the registry could not name: extensions ranked by + * file count, and declared dependencies that matched no entry. It is the whole + * reason an unmapped extension can be excluded from LOC without vanishing. + */ +function stackFromDetection(detected) { + const base = { registryVersion: detected?.registryVersion ?? STACK_REGISTRY_VERSION }; + if (!stackMeasured(detected)) { return { - ...base, total: unknown(result.reason), byLanguage: {}, files: null, skipped: 0, - complete: false, degraded: result.degraded ?? [], + ...base, + status: 'unknown', + reason: detected?.totalLines?.reason ?? 'not measured', + // Null, not `[]`: an empty list would state that this project declares no + // frameworks, which is a measurement nobody took (invariant 2). + items: null, + manifests: null, + nonSource: null, + unrecognized: null, + complete: false, + degraded: detected?.degraded ?? [], }; } return { ...base, - total: measured(total, { asOf, partial: result.complete === false }), - byLanguage, - files, - skipped, - complete: result.complete !== false, - degraded: result.degraded ?? [], + status: 'measured', + reason: null, + items: (detected.stack ?? []).map((row) => ({ ...row })), + manifests: (detected.manifests ?? []).map((row) => ({ ...row })), + nonSource: { ...(detected.nonSource ?? { files: null, bytes: null }) }, + unrecognized: { + extensions: (detected.unrecognized?.extensions ?? []).map((row) => ({ ...row })), + extensionsTotal: detected.unrecognized?.extensionsTotal ?? null, + dependencies: (detected.unrecognized?.dependencies ?? []).map((row) => ({ ...row })), + dependenciesTotal: detected.unrecognized?.dependenciesTotal ?? null, + }, + complete: detected.complete !== false, + degraded: detected.degraded ?? [], }; } +/** The not-measured LOC shape. `loc: false` skips the walk entirely, so every + * figure is unknown-with-reason — never a zero, which would claim an empty + * project (invariant 2). */ +const locNotMeasured = (reason) => locFromStack({ totalLines: unknown(reason) }); + +/** The not-measured stack shape, for the same reason. */ +const stackNotMeasured = (reason) => stackFromDetection({ totalLines: unknown(reason) }); + +/** + * Approximate lines of code under `root`, bucketed by language. + * + * A LOC-only projection of one `detectStack()` pass with the manifest reads + * switched off, for a caller that wants the count without the stack. Everything + * it inherits — never-follow-symlinks, depth and entry caps, the + * degrade-this-node-only failure mode, and which extensions are counted at all — + * comes from the shared walker and the registry, not from this module. + * + * @param {string} root + * @param {{ walk?: Function, limits?: object, asOf?: number|null, + * detect?: Function, fsImpl?: typeof fs }} [options] + * @returns {object} LocCount + */ +export function countLines(root, { + walk = walkTree, limits = {}, asOf = null, detect = detectStack, fsImpl = fs, +} = {}) { + return locFromStack(detect(root, { walk, limits, asOf, fsImpl, manifests: false })); +} + // ── bytes ───────────────────────────────────────────────────────────────────── /** One walked root as Measurements plus its newest mtime. `rootMeasurements` @@ -317,9 +364,10 @@ function missingProject(project, reason, presence = 'absent') { path: project.path, label: project.label, source: project.source ?? null, + hosts: Array.isArray(project.hosts) ? [...project.hosts] : null, remote: { status: 'unknown', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason }, - loc: { approximate: true, exclusions: [...LOC_EXCLUSIONS], total: unknown(reason), - byLanguage: {}, files: null, skipped: 0, complete: false, degraded: [] }, + loc: locNotMeasured(reason), + stack: stackNotMeasured(reason), presence, treeBytes: unknown(reason), treeFiles: unknown(reason), @@ -342,15 +390,18 @@ function notify(onProgress, payload) { /** * Measure one project. `walk` is the shared bounded walker; the project's path is - * the only thing discovery contributes (invariant 9). + * the only thing discovery contributes (invariant 9) — `hosts` rides along as + * attribution (which hosts saw this project), never as a measurement. * - * @param {{ path: string, label: string, source?: string }} project - * @param {{ walk?: Function, limits?: object, countLines?: Function, loc?: boolean, + * @param {{ path: string, label: string, source?: string, hosts?: string[] }} project + * @param {{ walk?: Function, limits?: object, detect?: Function, loc?: boolean, * asOf?: number|null, fsImpl?: typeof fs }} [options] + * `loc: false` skips the stack pass entirely — the expensive part of a project + * row — and both `loc` and `stack` then report unknown rather than empty. * @returns {object} ProjectFootprint */ export function measureProject(project, { - walk = walkTree, limits = {}, countLines: countLinesImpl = countLines, + walk = walkTree, limits = {}, detect = detectStack, loc = true, asOf = null, fsImpl = fs, } = {}) { const root = project.path; @@ -380,15 +431,19 @@ export function measureProject(project, { ? measured(0, { asOf }) : sumMeasurements(moduleRoots.map((dir) => walkNode(walk, dir, common).bytes), { asOf }); + // ONE detection pass, split into its two projections below: lines belong to + // languages, presence belongs to frameworks/SDKs/tools, and the tail names what + // neither could claim. + const detected = loc ? detect(root, { walk, limits, asOf, fsImpl }) : null; + return { path: root, label: project.label, source: project.source ?? null, + hosts: Array.isArray(project.hosts) ? [...project.hosts] : null, remote: projectRemote(root, { fsImpl }), - loc: loc - ? countLinesImpl(root, { walk, limits, asOf, fsImpl }) - : { approximate: true, exclusions: [...LOC_EXCLUSIONS], total: unknown('not measured'), - byLanguage: {}, files: null, skipped: 0, complete: false, degraded: [] }, + loc: detected ? locFromStack(detected) : locNotMeasured('not measured'), + stack: detected ? stackFromDetection(detected) : stackNotMeasured('not measured'), presence: tree.presence, treeBytes: tree.bytes, treeFiles: tree.files, @@ -407,20 +462,122 @@ export function measureProject(project, { }; } +/** everSeen / onDisk / gitRepos for an EXPLICITLY supplied catalog, which carries + * no existence facts of its own. Two lstats per row — negligible next to the + * tree walk that follows, and the alternative is a KPI that cannot be stated. */ +function summarizeCatalog(rows, fsImpl) { + let onDisk = 0; + let gitRepos = 0; + for (const row of rows) { + if (!row?.path) continue; + const node = statNode(row.path, { fsImpl }); + if (node.status === UNKNOWN || node.kind !== 'dir') continue; + onDisk += 1; + // A linked worktree's `.git` is a FILE, not a directory; checking only for a + // directory would undercount every worktree on the machine. + const git = statNode(path.join(row.path, '.git'), { fsImpl }); + if (git.status !== UNKNOWN && (git.kind === 'dir' || git.kind === 'file')) gitRepos += 1; + } + return { everSeen: rows.length, onDisk, gitRepos, unresolved: 0, complete: true }; +} + +/** A `discoverProjectSources()` PAYLOAD rather than a plain catalog array. + * Accepted wherever a catalog is, so a caller holding the payload — which is the + * only thing that carries everSeen/onDisk/gitRepos and the per-row `exists` flag + * — does not have to take it apart and lose them on the way in. */ +function isSourcesPayload(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Array.isArray(value.projects); +} + /** - * ProjectFootprint rows for every project in the shared discovery catalog. + * The machine-wide unrecognized tail: every extension and declared dependency the + * registry could not name, merged across projects and ranked. * - * @param {{ discover?: Function, walk?: Function, limits?: object, countLines?: Function, - * projects?: Array<{path: string, label: string}>|null, loc?: boolean, - * limit?: number|null, onProgress?: Function, now?: () => number, - * fsImpl?: typeof fs }} [options] + * Per project the tail is a curiosity; merged it is the to-do list a release + * closes, which is the whole reason an unmapped extension is excluded from LOC + * rather than guessed at. A total is stated only when it is a true distinct count + * — if any project's own list was capped, the merged count is a floor and says so + * with `null` rather than a smaller number presented as complete. + */ +function aggregateUnrecognized(rows) { + const extensions = new Map(); + const dependencies = new Map(); + let extensionsPartial = false; + let dependenciesPartial = false; + let measuredRows = 0; + + for (const row of rows) { + const tail = row?.stack?.unrecognized; + if (!tail) continue; + measuredRows += 1; + if (tail.extensionsTotal === null || tail.extensionsTotal > tail.extensions.length) { + extensionsPartial = true; + } + if (tail.dependenciesTotal === null || tail.dependenciesTotal > tail.dependencies.length) { + dependenciesPartial = true; + } + for (const entry of tail.extensions) { + const held = extensions.get(entry.ext); + if (held) { + held.files += entry.files; + held.bytes += entry.bytes; + held.projects += 1; + } else { + extensions.set(entry.ext, { ext: entry.ext, files: entry.files, bytes: entry.bytes, projects: 1 }); + } + } + for (const entry of tail.dependencies) { + const key = `${entry.manifest} ${entry.name.toLowerCase()}`; + const held = dependencies.get(key); + if (held) held.projects += 1; + else dependencies.set(key, { name: entry.name, manifest: entry.manifest, projects: 1 }); + } + } + + const rankedExtensions = [...extensions.values()] + .sort((a, b) => b.files - a.files || a.ext.localeCompare(b.ext)); + const rankedDependencies = [...dependencies.values()] + .sort((a, b) => b.projects - a.projects + || a.manifest.localeCompare(b.manifest) || a.name.localeCompare(b.name)); + + return { + projectsMeasured: measuredRows, + extensions: rankedExtensions.slice(0, SECTION_TAIL_EXTENSIONS), + extensionsTotal: extensionsPartial ? null : rankedExtensions.length, + dependencies: rankedDependencies.slice(0, SECTION_TAIL_DEPENDENCIES), + dependenciesTotal: dependenciesPartial ? null : rankedDependencies.length, + }; +} + +/** + * ProjectFootprint rows for every project that still exists on disk, plus the + * ever-seen / on-disk / git-repo counts the Summary KPI states together. + * + * The TABLE is the on-disk subset by necessity — a deleted project has no bytes + * and no lines to count — while `everSeen` keeps the deleted ones, because the + * count of projects this machine has touched is a different question from the + * count it can still measure. Rendering either number alone without saying which + * one it is would misreport both, which is why `method` ships with them. + * + * @param {{ discover?: Function, sources?: object|null, walk?: Function, limits?: object, + * detect?: Function, projects?: Array<{path: string, label: string}>|object|null, + * loc?: boolean, limit?: number|null, onProgress?: Function, + * now?: () => number, fsImpl?: typeof fs }} [options] + * `sources` is an already-resolved `discoverProjectSources()` payload, so a + * caller that shares discovery with another collector pays for the transcript + * sweep once. `projects` takes EITHER shape: an explicit catalog ARRAY, whose + * rows are measured exactly as given because the caller — not discovery — chose + * them and the on-disk filter therefore does not apply; or a discovery PAYLOAD, + * which is read exactly as `sources` is, on-disk filter and KPI counts included. * @returns {object} the ProjectFootprint section of a FootprintSnapshot */ export function collectProjects({ - discover = discoverRuvfloProjects, + discover = discoverProjectSources, + sources = null, walk = walkTree, limits = {}, - countLines: countLinesImpl = countLines, + detect = detectStack, projects = null, loc = true, limit = null, @@ -429,11 +586,34 @@ export function collectProjects({ fsImpl = fs, } = {}) { const asOf = now(); - let catalog; // Discovery is a candidate-path source; if it cannot run, this section reports // nothing rather than taking the rest of the snapshot down with it. let discoveryReason = null; - try { catalog = projects ?? discover(); } catch (error) { catalog = []; discoveryReason = error?.code ?? 'discovery failed'; } + let catalog; + let counts; + if (Array.isArray(projects)) { + catalog = projects; + counts = summarizeCatalog(projects, fsImpl); + } else { + try { + const payload = (isSourcesPayload(projects) ? projects : sources) ?? discover({ fsImpl }); + // Only projects that still exist can be walked, so only they become rows — + // the vanished ones survive in `everSeen`, not as unmeasurable table rows. + catalog = (payload?.projects ?? []).filter((project) => project?.exists); + counts = { + everSeen: payload?.everSeen ?? 0, + onDisk: payload?.onDisk ?? 0, + gitRepos: payload?.gitRepos ?? 0, + unresolved: payload?.unresolved ?? 0, + complete: payload?.complete !== false, + method: payload?.method ?? null, + sources: payload?.sources ?? null, + }; + } catch (error) { + catalog = []; + discoveryReason = error?.code ?? 'discovery failed'; + } + } const rows = Array.isArray(catalog) ? catalog : []; const selected = typeof limit === 'number' && limit >= 0 ? rows.slice(0, limit) : rows; @@ -441,17 +621,40 @@ export function collectProjects({ for (const project of selected) { if (!project?.path) continue; notify(onProgress, { scanned: out.length, total: selected.length, phase: 'project', path: project.path }); - out.push(measureProject(project, { walk, limits, countLines: countLinesImpl, loc, asOf, fsImpl })); + out.push(measureProject(project, { walk, limits, detect, loc, asOf, fsImpl })); } notify(onProgress, { scanned: out.length, total: selected.length, phase: 'done', path: null }); + // A count whose sweep hit an unreadable transcript or an unrecoverable project + // directory is a FLOOR, not a total — `partial` is what makes a surface render + // it as "≥ N" instead of quietly overstating certainty. + const partial = counts ? counts.complete === false : false; + const kpi = (value) => (discoveryReason ? unknown(discoveryReason) : measured(value, { asOf, partial })); + return { asOf, projects: out, - count: discoveryReason ? unknown(discoveryReason) : measured(rows.length, { asOf }), + // Retained under its original name for existing consumers; it has always + // meant "how many projects discovery found", which is now everSeen. + count: kpi(counts?.everSeen ?? 0), + everSeen: kpi(counts?.everSeen ?? 0), + onDisk: kpi(counts?.onDisk ?? 0), + gitRepos: kpi(counts?.gitRepos ?? 0), + unresolved: counts?.unresolved ?? 0, + method: counts?.method ?? null, + sources: counts?.sources ?? null, scanned: out.length, truncated: selected.length < rows.length, locMeasured: loc, - complete: !discoveryReason && selected.length === rows.length && out.every((row) => row.complete), + // Which catalog produced the language and stack facts in every row above. A + // figure that moves between releases can then be explained by the registry + // that changed rather than by the machine that did not. + registryVersion: STACK_REGISTRY_VERSION, + // The machine-wide to-do list: what the registry saw and could not name. + // Stated as `null` when nothing was scanned, because an empty tail from an + // unmeasured scan would read as "the registry knows everything here". + unrecognized: loc ? aggregateUnrecognized(out) : null, + complete: !discoveryReason && !partial && selected.length === rows.length + && out.every((row) => row.complete), }; } diff --git a/src/lib/footprint/snapshot.mjs b/src/lib/footprint/snapshot.mjs index 43825f4..4f776eb 100644 --- a/src/lib/footprint/snapshot.mjs +++ b/src/lib/footprint/snapshot.mjs @@ -12,10 +12,11 @@ // this module, because a failed read carries `sections: null` — there is // no numeric field to misread. // * The runtime census is never persisted (invariant 5). It is structurally -// impossible to write one here: `writeSnapshot` serializes only the four -// section keys in SNAPSHOT_SECTIONS, so a caller that hands over a census -// by mistake silently drops it rather than replaying a stale process table -// as liveness. +// impossible to write one here: `writeSnapshot` serializes only the section +// keys in SNAPSHOT_SECTIONS, so a caller that hands over a census by +// mistake silently drops it rather than replaying a stale process table as +// liveness. That allow-list is the enforcement — adding a deep section +// means adding it below, and `runtime` may never be one of them. import fs from 'node:fs'; import path from 'node:path'; import { configDir } from '../paths.mjs'; @@ -28,9 +29,13 @@ import { CARRIED_FORWARD, MEASURED } from './walk.mjs'; * replaces it. */ export const SNAPSHOT_SCHEMA_VERSION = 1; -/** The four deep-tier sections, in render order. This list is also the write - * filter — see the header note on the runtime census. */ -export const SNAPSHOT_SECTIONS = Object.freeze(['install', 'storage', 'catalog', 'projects']); +/** The deep-tier sections, in collection order. This list is also the write + * filter — see the header note on the runtime census. A section absent from a + * previously written snapshot reads as never-measured rather than empty, so + * extending this list does not invalidate snapshots taken before it. */ +export const SNAPSHOT_SECTIONS = Object.freeze([ + 'install', 'storage', 'catalog', 'projects', 'consumers', +]); /** How old a deep scan gets before the UI nudges for a rescan. Deliberately a * nudge and not a trigger: ADR-0025's freshness policy is manual-rescan-only, @@ -166,7 +171,8 @@ export function summarizeCompleteness(sections = {}) { * returns an outcome instead of throwing, and the caller keeps the in-memory * result it just measured. * - * @param {object} sections the four deep sections; anything else is dropped + * @param {object} sections the deep sections named by SNAPSHOT_SECTIONS; + * anything else — the runtime census above all — is dropped * @param {{ file?: string, fsImpl?: typeof fs, now?: number, asOf?: number }} [options] * @returns {{ ok: boolean, file: string, asOf: number, error: string|null }} */ diff --git a/src/lib/footprint/stack-detect.mjs b/src/lib/footprint/stack-detect.mjs new file mode 100644 index 0000000..5f7b290 --- /dev/null +++ b/src/lib/footprint/stack-detect.mjs @@ -0,0 +1,582 @@ +// Stack detection over ONE project directory: which languages hold its lines, +// which frameworks / SDKs / tools it declares, and — the part that makes the +// registry self-improving — everything it saw and could NOT name (ADR-0025 §4, +// docs/ddd/machine-footprint.md "Project footprint"). +// +// LINES vs PRESENCE is the load-bearing distinction. `languages` carry a line +// count because a file extension is what a line belongs to. `stack` entries carry +// PRESENCE ONLY and are never given a `lines` field: react does not own lines, the +// .tsx files do, and stacking both on one proportional bar would count the same +// bytes twice. Nothing downstream can make that mistake by accident because the +// number simply is not in the payload. +// +// THE UNRECOGNIZED TAIL IS THE POINT. An extension the registry does not map is +// never counted as lines — most unmapped extensions on a real machine are binaries +// and data — so instead it is tallied BY NAME, as is every declared dependency that +// matched no entry. That converts the usual silent "Other" slice into a to-do list +// a release can close. +// +// Which is why the tail has exactly one job and admits only what belongs in it. An +// extension the registry has already RULED OUT (`.png`, `.sqlite` — the registry's +// non-source list) is a stated exclusion, not a gap, and is counted separately. +// A key that is not shaped like an extension at all (`.2026-08-06` from a rotated +// log) collapses into a named bucket. Without both filters the tail fills with +// noise and stops being read, which is the same failure as not having one. +// +// MANIFESTS ARE READ SHALLOWLY, AND ONLY FOR NAMES. Reading a project's own +// manifests extends the metadata-only rule (invariant 1) exactly as far as +// `.git/config`'s remote URL and `.git/worktrees/*/gitdir` already do: a bounded +// read of a declaration file, parsed for dependency KEYS and nothing else. Nothing +// here evaluates, executes, resolves or fetches anything — mix.exs and build.gradle +// are Elixir and Groovy source, and they are scanned with regexes, never run. The +// depth bound is not cosmetic: a scan of this machine found 1884 Cargo.toml files, +// nearly all of them inside cargo's registry cache, so a deep manifest search would +// report a dependency's dependencies as the project's own. +// +// The walk is the shared bounded walker from walk.mjs, so this module inherits the +// never-follow-symlinks rule, the entry/depth caps, and the degrade-this-node-only +// failure mode (invariant 6) rather than reimplementing them. +import fs from 'node:fs'; +import path from 'node:path'; +import { walkTree, measured, unknown } from './walk.mjs'; +import { + STACK_REGISTRY_VERSION, dependencyEntry, isNonSourceExtension, languageForExtension, + languageForFilename, manifestKindFor, registryStats, signatureEntries, +} from './stack-registry.mjs'; + +/** Vendored / generated / virtual-env trees. Excluded from the scan entirely: + * they are real bytes on disk (projects.mjs still counts them in treeBytes) but + * they are not lines the user wrote, and their manifests are not this project's + * declarations. Kept byte-identical to projects.mjs's list so the two agree on + * what "the user's code" means. */ +export const EXCLUDED_DIRS = new Set([ + 'node_modules', '.git', 'vendor', 'third_party', 'thirdparty', 'bower_components', + 'dist', 'build', 'out', 'target', '.next', '.nuxt', '.svelte-kit', 'coverage', + '.venv', 'venv', '__pycache__', '.tox', '.mypy_cache', '.pytest_cache', + '.gradle', '.idea', '.vscode', 'Pods', '.terraform', '.cache', '.turbo', +]); + +/** Machine-generated manifests: text, enormous, and nobody's line count. Skipped + * before the tail too — a lockfile is not an unrecognized extension, it is a + * deliberate exclusion, and listing it as a to-do would be noise. */ +export const EXCLUDED_FILES = new Set([ + 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'npm-shrinkwrap.json', + 'Cargo.lock', 'poetry.lock', 'Gemfile.lock', 'composer.lock', 'go.sum', 'flake.lock', +]); + +/** Source trees nest far shallower than dependency trees. */ +export const STACK_MAX_DEPTH = 12; +/** A project's own manifests plus one monorepo package level + * (`packages//package.json` sits at 3). Vendored trees are already pruned + * by EXCLUDED_DIRS, so this bound is about monorepo shape, not safety alone. */ +export const MANIFEST_MAX_DEPTH = 3; +/** `.github/workflows` is the deepest signature worth looking for. */ +export const SIGNATURE_MAX_DEPTH = 3; +/** Files above this are minified bundles, fixtures or data dumps far more often + * than hand-written source. Skipped and reported, never counted. */ +export const MAX_FILE_BYTES = 2 * 1024 * 1024; +/** A manifest larger than this is generated, not authored; refused rather than + * read, so one pathological file cannot dominate a scan. */ +export const MANIFEST_MAX_BYTES = 512 * 1024; + +const READ_CHUNK = 64 * 1024; +const MAX_MANIFESTS = 64; +const MAX_DEPS_PER_MANIFEST = 500; +const MAX_SIGNATURE_PATHS = 4000; +const MAX_TAIL_KEYS = 4000; +const TAIL_EXTENSIONS = 40; +const TAIL_DEPENDENCIES = 50; + +/** The exclusion list every surface must be able to state alongside the figure + * (invariant 11). Deliberate exclusions are a scope, not a failed measurement, so + * they never mark a count partial — which is exactly why they ship attached to it. */ +export const STACK_EXCLUSIONS = Object.freeze([ + ...[...EXCLUDED_DIRS].sort().map((dir) => `${dir}/`), + ...[...EXCLUDED_FILES].sort(), + 'files with a known non-source extension (images, media, archives, binaries, stores)', + 'files without a registry-recognized source extension (listed as the unrecognized tail)', + 'files containing NUL bytes (binary)', + `files larger than ${MAX_FILE_BYTES} bytes`, +]); + +/** The tail is keyed by extension — but only when the extension is shaped like + * one. `path.extname('debug.2026-08-06')` is `.2026-08-06`, and one rotated log + * directory would otherwise mint a thousand single-file "extensions" and drown + * the to-do list it exists to be. Everything else collapses into two named + * buckets, which is honest and stays countable. */ +const TAIL_OTHER = '(other)'; +const TAIL_NONE = '(no extension)'; +// Digits are allowed (`.ps1`, `.f90`, `.mp3`) but separators are not: no real +// source extension contains a hyphen or an underscore, while every rotated, +// versioned or quarantined filename does (`.corrupt-4585`, `.2026-08-06`). +const PLAUSIBLE_EXTENSION = /^\.[a-z][a-z0-9+#]{0,10}$/; + +function tailKey(lowerName) { + const ext = path.extname(lowerName); + if (!ext) return TAIL_NONE; + return PLAUSIBLE_EXTENSION.test(ext) ? ext : TAIL_OTHER; +} + +// ── lines ───────────────────────────────────────────────────────────────────── + +/** A file a cloud provider has evicted: real size, zero allocated blocks. Reading + * one is not slow, it is UNBOUNDED — the open blocks in the kernel until the + * provider faults the bytes back in, and with the provider signed out or offline + * that never happens and no timeout fires. Measured on this machine: a 4KB + * dataless Dropbox `.yml` held a synchronous read for over 15 minutes, which is + * the whole deep scan and (the collectors being synchronous) the whole server. + * + * So placeholders are never opened. They are skipped like any other unreadable + * file — the project's line count is a floor and the walk reports incomplete, + * which is invariant 3's "unknown, never 0" rather than a fabricated zero. + * + * Only an explicit 0 counts. `undefined` means the stat did not carry blocks (a + * test shim, a platform that omits it), and guessing from a missing field would + * skip real files. Sparse files also report fewer blocks than their size implies + * but never zero-with-content, so they are unaffected. */ +const isCloudPlaceholder = (bytes, blocks) => blocks === 0 && bytes > 0; + +/** Count newlines in one file through a fixed buffer. Returns null when the file + * is binary (a NUL byte in the first chunk) or unreadable — never 0, which would + * claim an empty file. Each chunk is counted and immediately overwritten; no file + * content is retained past this function or emitted anywhere. */ +function countFileLines(file, size, fsImpl) { + let fd; + try { fd = fsImpl.openSync(file, 'r'); } catch { return null; } + try { + const buffer = Buffer.allocUnsafe(READ_CHUNK); + let lines = 0; + let read = 0; + let lastByte = 0; + let offset = 0; + let first = true; + while ((read = fsImpl.readSync(fd, buffer, 0, READ_CHUNK, offset)) > 0) { + if (first) { const nul = buffer.indexOf(0); if (nul >= 0 && nul < read) return null; } + first = false; + for (let i = 0; i < read; i++) if (buffer[i] === 0x0a) lines++; + lastByte = buffer[read - 1]; + offset += read; + } + // A final line with no trailing newline still counts as a line. + if (size > 0 && lastByte !== 0x0a) lines++; + return lines; + } catch { return null; } + finally { try { fsImpl.closeSync(fd); } catch { /* fd already gone */ } } +} + +// ── manifest parsing (names and keys only — nothing is evaluated) ───────────── + +/** A dependency resolved from inside this repository is not a stack fact — it is + * the project depending on itself. Workspace and path protocols are the one thing + * a dependency's VALUE is read for; nothing else about it is inspected. */ +const LOCAL_PROTOCOL = /^(workspace|file|link|portal|path):/i; + +const jsonDependencyKeys = (source, sections) => { + let doc; + try { doc = JSON.parse(source); } catch { return null; } + const names = []; + for (const section of sections) { + const block = doc?.[section]; + if (!block || typeof block !== 'object' || Array.isArray(block)) continue; + for (const [name, version] of Object.entries(block)) { + if (typeof version === 'string' && LOCAL_PROTOCOL.test(version)) continue; + names.push(name); + } + } + return names; +}; + +/** TOML keys inside any section whose header ends in `dependencies`, plus the + * `[dependencies.]` table form. Line-oriented on purpose: a real TOML + * parser is a dependency this context does not have and does not need for keys. */ +function tomlDependencyKeys(source) { + const names = []; + let inDeps = false; + for (const line of source.split(/\r?\n/)) { + const header = line.match(/^\s*\[\s*([^\]]+?)\s*\]\s*$/); + if (header) { + const section = header[1].replace(/["']/g, ''); + const table = section.match(/(?:^|\.)(?:dev-|build-|dependency-)?dependencies\.(.+)$/); + if (table) { names.push(table[1].split('.')[0]); inDeps = false; continue; } + // `packages` covers Pipfile, whose section is spelled differently from + // every other TOML manifest's. + inDeps = /(?:^|\.)(?:dev-|build-|dependency-|optional-)?(?:dependencies|packages)$/ + .test(section) || /(?:^|\.)dependency-groups$/.test(section); + continue; + } + if (!inDeps) continue; + // `sibling = { path = "crates/…" }` is a workspace member, not a third-party + // dependency; counting it would list a Cargo workspace's own crates as its stack. + if (/\bpath\s*=/.test(line)) continue; + const key = line.match(/^\s*(?:"([^"]+)"|'([^']+)'|([A-Za-z0-9_.-]+))\s*=/); + if (key) names.push(key[1] ?? key[2] ?? key[3]); + } + return names; +} + +/** Quoted requirement strings inside a `dependencies = [ … ]` array (PEP 621 and + * the optional-dependency groups that share its shape). */ +function tomlDependencyArrays(source) { + const names = []; + const arrays = source.matchAll(/dependencies\s*=\s*\[([\s\S]*?)\]/g); + for (const array of arrays) { + for (const quoted of array[1].matchAll(/["']([^"']+)["']/g)) names.push(quoted[1]); + } + return names; +} + +/** PyPI treats `_` and `-` as the same character and casing as insignificant. */ +const pythonName = (raw) => { + const head = String(raw).trim().match(/^[A-Za-z0-9._-]+/); + return head ? head[0].toLowerCase().replace(/_/g, '-') : null; +}; + +function goModules(source) { + const names = []; + for (const line of source.split(/\r?\n/)) { + if (line.includes('// indirect')) continue; // transitive, not this project's stack + const match = line.match(/^\s*(?:require\s+)?([a-z0-9][^\s(]*\.[a-z]{2,}\/[^\s]+)\s+v/i); + if (match) names.push(match[1]); + } + return names; +} + +/** Maven coordinates and Gradle dependency strings alike: every quoted token that + * looks like a coordinate contributes its group and its artifact, because the + * registry matches whichever half is the recognizable one. */ +function jvmCoordinates(source) { + const names = []; + for (const tag of source.matchAll(/<(?:groupId|artifactId)>\s*([^<\s]+)\s*<\//g)) names.push(tag[1]); + for (const quoted of source.matchAll(/["']([A-Za-z][\w.-]*(?::[\w.$-]+){0,2})["']/g)) { + for (const part of quoted[1].split(':')) { + if (part && !/^\d/.test(part)) names.push(part); + } + } + return names; +} + +/** Top-level `dependencies:` / `dev_dependencies:` keys in a pubspec. */ +function yamlDependencyKeys(source) { + const names = []; + let inDeps = false; + for (const line of source.split(/\r?\n/)) { + if (/^[A-Za-z_]/.test(line)) { + inDeps = /^(dependencies|dev_dependencies|dependency_overrides):\s*$/.test(line); + continue; + } + if (!inDeps) continue; + const key = line.match(/^\s{2}([A-Za-z0-9_.-]+):/); + if (key) names.push(key[1]); + } + return names; +} + +/** + * Dependency names declared by one manifest. Pure: no I/O, no evaluation — every + * parser here is JSON.parse or a regex scan, so a manifest that is source code + * (mix.exs, build.gradle) is read as text and never run. + * + * Returns null when the source could not be parsed at all, which the caller + * reports as a degraded manifest rather than as "declares nothing". + * + * @param {string} kind one of MANIFEST_KINDS + * @param {string} source raw manifest text + * @returns {string[]|null} + */ +export function parseManifestDependencies(kind, source) { + const text = String(source ?? ''); + switch (kind) { + case 'npm': + return jsonDependencyKeys(text, + ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']); + case 'composer': + return jsonDependencyKeys(text, ['require', 'require-dev']); + case 'cargo': + return tomlDependencyKeys(text); + case 'gomod': + return goModules(text); + case 'python': { + // pyproject.toml / Pipfile and requirements.txt share a manifest kind and + // nothing else, so the two shapes are read exclusively: running the + // line-per-requirement scan over a TOML file turns classifier strings into + // imaginary dependencies. A leading `[` on any line is the discriminator — + // no requirement line can start with one. + if (/^\s*\[/m.test(text)) { + return [...tomlDependencyKeys(text), ...tomlDependencyArrays(text)] + .map(pythonName).filter(Boolean); + } + const raw = []; + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('-')) continue; + raw.push(trimmed); + } + return raw.map(pythonName).filter(Boolean); + } + case 'jvm': + return jvmCoordinates(text); + case 'pub': + return yamlDependencyKeys(text); + case 'hex': + return [...text.matchAll(/\{\s*:([a-z0-9_]+)\s*,/g)].map((m) => m[1]); + case 'rubygems': + return [...text.matchAll(/^\s*gem\s+["']([^"']+)["']/gm)].map((m) => m[1]); + default: + return null; + } +} + +/** One manifest, bounded and read for keys only. `bytes` comes from the walk's + * own lstat, so an oversized manifest is refused without ever being opened. */ +function readManifest(file, kind, bytes, fsImpl) { + if (bytes > MANIFEST_MAX_BYTES) { + return { path: file, kind, status: 'skipped', reason: 'manifest larger than cap', names: [] }; + } + let source; + try { source = fsImpl.readFileSync(file, 'utf8'); } catch (error) { + return { path: file, kind, status: 'unreadable', reason: error?.code ?? 'io', names: [] }; + } + const names = parseManifestDependencies(kind, source); + if (names === null) { + return { path: file, kind, status: 'unparsed', reason: 'EPARSE', names: [] }; + } + return { + path: file, + kind, + status: 'read', + reason: null, + names: names.slice(0, MAX_DEPS_PER_MANIFEST), + truncated: names.length > MAX_DEPS_PER_MANIFEST, + }; +} + +// ── detection ───────────────────────────────────────────────────────────────── + +const KIND_ORDER = { framework: 0, sdk: 1, tool: 2 }; +const rel = (root, target) => path.relative(root, target).split(path.sep).join('/').toLowerCase(); + +/** The shape returned when the project root itself could not be walked. Every + * figure is unknown-with-reason; none of them is 0, because nothing was measured + * (invariant 2). */ +function unmeasured(reason, asOf) { + return { + registryVersion: STACK_REGISTRY_VERSION, + asOf, + approximate: true, + exclusions: [...STACK_EXCLUSIONS], + languages: [], + totalLines: unknown(reason), + stack: [], + unrecognized: { extensions: [], extensionsTotal: 0, dependencies: [], dependenciesTotal: 0 }, + nonSource: { files: null, bytes: null }, + manifests: [], + files: null, + skipped: 0, + complete: false, + degraded: [], + }; +} + +/** + * Detect the stack of one project directory. + * + * One walk answers all three questions: files are line-counted into their + * language bucket or tallied into the unrecognized tail, shallow manifests and + * signature paths are collected on the way past, and the manifests are read after + * the walk so the bounded traversal is never interleaved with file reads. + * + * @param {string} root the project's directory + * @param {{ walk?: Function, limits?: object, maxDepth?: number, + * manifestDepth?: number, signatureDepth?: number, manifests?: boolean, + * asOf?: number|null, fsImpl?: typeof fs }} [options] + * @returns {object} `{ registryVersion, languages[{id,name,ecosystem,colorSlot,lines, + * files}], totalLines: Measurement, stack[{id,kind,name,ecosystem,via}], + * unrecognized: { extensions[{ext,files,bytes}], dependencies[{name,manifest}] }, + * nonSource, manifests, exclusions, complete, degraded }` + */ +export function detectStack(root, { + walk = walkTree, + limits = {}, + maxDepth = STACK_MAX_DEPTH, + manifestDepth = MANIFEST_MAX_DEPTH, + signatureDepth = SIGNATURE_MAX_DEPTH, + manifests: readManifests = true, + asOf = null, + fsImpl = fs, +} = {}) { + const lines = new Map(); // language id → { entry, lines, files } + const tail = new Map(); // extension (or bare filename) → { files, bytes } + const manifestFiles = []; // { file, kind, bytes } + const seenFiles = new Set(); // shallow basenames, for signature matching + const seenPaths = new Set(); // shallow root-relative file paths + const seenDirs = new Set(); // shallow root-relative directory paths + let skipped = 0; + let files = 0; + let tailTruncated = false; + let nonSourceFiles = 0; + let nonSourceBytes = 0; + + const note = (set, value) => { if (set.size < MAX_SIGNATURE_PATHS) set.add(value); }; + + const result = walk(root, { + maxDepth, ...limits, fsImpl, + skipDir: (dir, name, depth) => { + // Recorded BEFORE the skip decision: `.terraform` and `.git` are excluded + // from the scan and are still evidence of what this project uses. + if (depth <= signatureDepth) note(seenDirs, rel(root, dir)); + return EXCLUDED_DIRS.has(name); + }, + // Rejected files still consume the entry budget but do no work — and, being + // deliberate exclusions, they never reach the unrecognized tail either. + acceptFile: (name) => !EXCLUDED_FILES.has(name), + onFile: ({ file, name, bytes, blocks, depth }) => { + const lower = name.toLowerCase(); + const placeholder = isCloudPlaceholder(bytes, blocks); + if (depth <= signatureDepth) { note(seenFiles, lower); note(seenPaths, rel(root, file)); } + // A placeholder manifest is still evidence the project HAS that manifest — + // its name was noted above. Only its contents are out of reach, so it is + // never queued for the read pass. + if (readManifests && !placeholder + && depth <= manifestDepth && manifestFiles.length < MAX_MANIFESTS) { + const kind = manifestKindFor(name); + if (kind) manifestFiles.push({ file, kind, bytes }); + } + + const entry = languageForFilename(name) ?? languageForExtension(path.extname(name)); + if (!entry) { + // A STATED non-source extension is a decision, not a gap: it is named in + // `exclusions` and counted here, and it never joins the tail — otherwise + // .png and .sqlite would bury the extensions the registry should learn. + if (isNonSourceExtension(path.extname(lower))) { + nonSourceFiles += 1; + nonSourceBytes += bytes; + return; + } + // A manifest is recognized — it simply holds declarations rather than + // lines. `go.mod` reported as an unrecognized `.mod` extension would be + // the tail contradicting the parser that just read it. + if (manifestKindFor(name)) return; + const key = tailKey(lower); + const row = tail.get(key); + if (row) { row.files += 1; row.bytes += bytes; } else if (tail.size < MAX_TAIL_KEYS) { + tail.set(key, { files: 1, bytes }); + } else tailTruncated = true; + return; + } + if (bytes > MAX_FILE_BYTES) { skipped++; return; } + if (placeholder) { skipped++; return; } + const counted = countFileLines(file, bytes, fsImpl); + if (counted === null) { skipped++; return; } + const bucket = lines.get(entry.id); + if (bucket) { bucket.lines += counted; bucket.files += 1; } else { + lines.set(entry.id, { entry, lines: counted, files: 1 }); + } + files++; + }, + }); + + if (result.status === 'unknown') return unmeasured(result.reason ?? 'unreadable', asOf); + + // Manifests are read after the walk: the traversal stays a pure metadata pass, + // and a slow read cannot hold the walker's entry budget open. + const manifestRows = []; + const stack = new Map(); + const unrecognizedDeps = new Map(); + for (const { file, kind, bytes } of manifestFiles) { + const reading = readManifest(file, kind, bytes, fsImpl); + manifestRows.push({ + path: reading.path, kind: reading.kind, status: reading.status, reason: reading.reason, + }); + for (const name of reading.names) { + const entry = dependencyEntry(kind, name); + if (entry) { + if (!stack.has(entry.id)) stack.set(entry.id, { entry, via: kind }); + continue; + } + const key = `${kind} ${name.toLowerCase()}`; + if (!unrecognizedDeps.has(key)) unrecognizedDeps.set(key, { name, manifest: kind }); + } + } + + for (const entry of signatureEntries()) { + const { files: names, dirs, filePrefixes } = entry.match; + const hit = names.find((name) => seenFiles.has(name) || seenPaths.has(name)) + ?? dirs.find((dir) => seenDirs.has(dir)) + ?? (filePrefixes.length + ? [...seenFiles].find((name) => filePrefixes.some((prefix) => name.startsWith(prefix))) + : undefined); + if (hit !== undefined && !stack.has(entry.id)) stack.set(entry.id, { entry, via: hit }); + } + + const languages = [...lines.values()] + .map(({ entry, lines: count, files: fileCount }) => ({ + id: entry.id, + name: entry.name, + ecosystem: entry.ecosystem, + colorSlot: entry.colorSlot, + lines: count, + files: fileCount, + })) + .sort((a, b) => b.lines - a.lines || a.id.localeCompare(b.id)); + + const total = languages.reduce((sum, row) => sum + row.lines, 0); + const extensions = [...tail.entries()] + .map(([ext, row]) => ({ ext, files: row.files, bytes: row.bytes })) + .sort((a, b) => b.files - a.files || a.ext.localeCompare(b.ext)); + const dependencies = [...unrecognizedDeps.values()] + .sort((a, b) => a.manifest.localeCompare(b.manifest) || a.name.localeCompare(b.name)); + + return { + registryVersion: STACK_REGISTRY_VERSION, + asOf, + approximate: true, + exclusions: [...STACK_EXCLUSIONS], + languages, + // The Measurement carries what the array cannot: a walk that hit a cap or an + // unreadable subtree makes this a floor, which every surface renders as "≥ N". + totalLines: measured(total, { asOf, partial: result.complete === false }), + // PRESENCE ONLY — deliberately no `lines` field; see this file's header. + stack: [...stack.values()] + .map(({ entry, via }) => ({ + id: entry.id, kind: entry.kind, name: entry.name, ecosystem: entry.ecosystem, via, + })) + .sort((a, b) => (KIND_ORDER[a.kind] - KIND_ORDER[b.kind]) || a.name.localeCompare(b.name)), + unrecognized: { + // Totals sit next to the capped lists so a truncated tail is a stated + // number, never a quietly shorter one. + extensions: extensions.slice(0, TAIL_EXTENSIONS), + extensionsTotal: tailTruncated ? null : extensions.length, + dependencies: dependencies.slice(0, TAIL_DEPENDENCIES), + dependenciesTotal: dependencies.length, + }, + // Not a gap and not lines: the bytes this project holds in things the + // registry has already ruled out as source. + nonSource: { files: nonSourceFiles, bytes: nonSourceBytes }, + manifests: manifestRows, + files, + skipped, + complete: result.complete !== false && manifestRows.every((row) => row.status === 'read'), + degraded: result.degraded ?? [], + }; +} + +/** The liner note a panel prints under a changed number: what was counted, by + * which registry, and what the tail still owes. Returned as parts rather than a + * sentence so the UI owns the typography and this module owns the facts. */ +export function stackProvenance(detected) { + const stats = registryStats(); + return Object.freeze({ + registryVersion: stats.version, + registryEntries: stats.entries, + recognizedExtensions: stats.extensions, + nonSourceExtensions: stats.nonSourceExtensions, + nonSourceFiles: detected?.nonSource?.files ?? null, + languages: detected?.languages?.length ?? 0, + stackItems: detected?.stack?.length ?? 0, + unrecognizedExtensions: detected?.unrecognized?.extensionsTotal ?? null, + unrecognizedDependencies: detected?.unrecognized?.dependenciesTotal ?? null, + manifestsRead: (detected?.manifests ?? []).filter((row) => row.status === 'read').length, + manifestsSeen: detected?.manifests?.length ?? 0, + approximate: true, + }); +} diff --git a/src/lib/footprint/stack-registry.mjs b/src/lib/footprint/stack-registry.mjs new file mode 100644 index 0000000..c471089 --- /dev/null +++ b/src/lib/footprint/stack-registry.mjs @@ -0,0 +1,610 @@ +// The stack registry — the curated catalog of every language, framework, SDK and +// tool this kit can name (ADR-0025 §4, docs/ddd/machine-footprint.md "Project +// footprint"). Detection lives next door in `stack-detect.mjs`; this file only +// says what a match MEANS. +// +// PURE DATA, NO I/O — the same discipline as src/lib/dashboard/about-directory.mjs. +// Importing this module has no side effects, so a unit test can assert the whole +// catalog without a machine to measure and a reviewer can read a release's stack +// vocabulary as a diff. +// +// LINES BELONG TO LANGUAGES, ONLY. A 'language' entry owns extensions and therefore +// owns lines. A framework / sdk / tool entry is PRESENCE ONLY and must never be +// given a line count: react does not own lines, the .tsx files do, and putting both +// on one proportional bar would count the same bytes twice. +// +// AN UNMAPPED EXTENSION IS NEVER COUNTED. On a real machine most unmapped +// extensions are binaries and data (.jsonl, .png, .jar, .dll), not source, so +// guessing would inflate every total. The unmapped tail is instead surfaced BY NAME +// by stack-detect.mjs — which turns "Other" from a shrug into a to-do list this +// registry shrinks release by release. +// +// VERSIONED because it is an artifact that grows. A snapshot records the version +// that produced it, so a figure that moved between releases can be explained by the +// registry that changed rather than by the machine that did not. + +/** Bump on every entry change. Date-ordinal so a snapshot's provenance reads + * without a lookup table; consumers compare for equality, never for ordering. */ +export const STACK_REGISTRY_VERSION = '2026.08.1'; + +/** The closed set of ecosystem tags. Closed on purpose: an open string field + * becomes a synonym pile ('js' / 'javascript' / 'node') that no grouping can + * reconcile. Adding a tag is a registry change, reviewed with the entry. */ +export const ECOSYSTEMS = Object.freeze([ + 'node', 'web', 'rust', 'go', 'python', 'ruby', 'php', 'jvm', 'dotnet', 'apple', + 'beam', 'native', 'functional', 'shell', 'docs', 'data', 'infra', 'ai', 'mobile', + 'gamedev', 'other', +]); + +/** Palette SLOTS, not colours. The registry names the token; the stylesheet owns + * the hue — the same split as about-directory's `hue`, so a theme change never + * requires a data change. */ +export const COLOR_SLOTS = Object.freeze([ + 'js', 'ts', 'rust', 'go', 'python', 'ruby', 'php', 'jvm', 'dotnet', 'apple', + 'beam', 'native', 'functional', 'shell', 'markup', 'docs', 'data', 'infra', 'other', +]); + +/** Manifest families. A dependency name only means something inside one of these: + * `openai` is one package on npm and a different one on PyPI, and an entry says + * which manifests its names are valid in. */ +export const MANIFEST_KINDS = Object.freeze([ + 'npm', 'cargo', 'gomod', 'python', 'jvm', 'pub', 'hex', 'composer', 'rubygems', +]); + +/** Manifest FILENAME → manifest kind. `manifestKindFor` is the accessor; this map + * is exported so a caller can state which filenames it looked for. */ +export const MANIFEST_FILES = Object.freeze({ + 'package.json': 'npm', + 'cargo.toml': 'cargo', + 'go.mod': 'gomod', + 'pyproject.toml': 'python', + 'requirements.txt': 'python', + pipfile: 'python', + 'pom.xml': 'jvm', + 'build.gradle': 'jvm', + 'build.gradle.kts': 'jvm', + 'pubspec.yaml': 'pub', + 'mix.exs': 'hex', + 'composer.json': 'composer', + gemfile: 'rubygems', +}); + +// ── entry constructors ──────────────────────────────────────────────────────── +// +// `kind` says WHAT a thing is; `match.by` says HOW it is found. Those are two +// different questions and conflating them is why "is ESLint a framework?" has no +// good answer: ESLint is a tool that happens to be found in a manifest. The only +// rule that binds is the one above — `by: 'extension'` entries carry lines, every +// other entry carries presence. + +const freezeList = (list) => Object.freeze([...list]); +const lower = (list) => Object.freeze(list.map((s) => s.toLowerCase())); + +/** A language: extensions (and occasionally bare filenames — a Makefile has lines + * and no extension) mapped to one bucket, with the palette slot it renders in. */ +const lang = (id, name, ecosystem, colorSlot, extensions, filenames = []) => Object.freeze({ + id, + kind: 'language', + name, + ecosystem, + colorSlot, + match: Object.freeze({ by: 'extension', extensions: lower(extensions), filenames: lower(filenames) }), +}); + +/** A manifest-detected dependency. `manifests` is a list because one project can + * be the same thing under two ecosystems (the OpenAI SDK is npm AND PyPI), and + * splitting it into two entries would double it in every UI that lists names. */ +const dep = (id, kind, name, ecosystem, manifests, names, prefixes = []) => Object.freeze({ + id, + kind, + name, + ecosystem, + match: Object.freeze({ + by: 'dependency', + manifests: freezeList(manifests), + names: lower(names), + prefixes: lower(prefixes), + }), +}); + +/** A file/directory signature. `files` and `dirs` match basenames and + * root-relative paths respectively; `filePrefixes` covers the families that vary + * by suffix (`.eslintrc.json`, `docker-compose.prod.yml`). */ +const sig = (id, name, ecosystem, { files = [], dirs = [], filePrefixes = [] }) => Object.freeze({ + id, + kind: 'tool', + name, + ecosystem, + match: Object.freeze({ + by: 'signature', + files: lower(files), + dirs: lower(dirs), + filePrefixes: lower(filePrefixes), + }), +}); + +// ── languages ───────────────────────────────────────────────────────────────── +// +// Seeded from the LANGUAGES map projects.mjs shipped (~65 extensions) and widened. +// Bucket ids are kept byte-identical where they existed ('javascript', 'config', +// 'objective-c', …) so a snapshot taken before this registry still joins. + +const LANGUAGE_ENTRIES = [ + lang('javascript', 'JavaScript', 'node', 'js', ['.js', '.mjs', '.cjs', '.jsx']), + lang('typescript', 'TypeScript', 'node', 'ts', ['.ts', '.tsx', '.mts', '.cts']), + lang('coffeescript', 'CoffeeScript', 'node', 'js', ['.coffee']), + lang('rust', 'Rust', 'rust', 'rust', ['.rs']), + lang('go', 'Go', 'go', 'go', ['.go']), + lang('python', 'Python', 'python', 'python', ['.py', '.pyi', '.pyx']), + lang('ruby', 'Ruby', 'ruby', 'ruby', ['.rb', '.rake', '.gemspec']), + lang('php', 'PHP', 'php', 'php', ['.php', '.phtml']), + lang('java', 'Java', 'jvm', 'jvm', ['.java']), + lang('kotlin', 'Kotlin', 'jvm', 'jvm', ['.kt', '.kts']), + lang('scala', 'Scala', 'jvm', 'jvm', ['.scala', '.sc']), + lang('groovy', 'Groovy', 'jvm', 'jvm', ['.groovy', '.gradle']), + lang('clojure', 'Clojure', 'jvm', 'jvm', ['.clj', '.cljs', '.cljc', '.edn']), + lang('csharp', 'C#', 'dotnet', 'dotnet', ['.cs', '.csx']), + lang('fsharp', 'F#', 'dotnet', 'dotnet', ['.fs', '.fsi', '.fsx']), + lang('visual-basic', 'Visual Basic', 'dotnet', 'dotnet', ['.vb']), + lang('razor', 'Razor', 'dotnet', 'dotnet', ['.cshtml', '.razor']), + lang('swift', 'Swift', 'apple', 'apple', ['.swift']), + // `.m` is Objective-C here, as it was in projects.mjs: MATLAB is the rarer + // reading of that extension on a machine running agent CLIs. + lang('objective-c', 'Objective-C', 'apple', 'apple', ['.m', '.mm']), + lang('c', 'C', 'native', 'native', ['.c', '.h']), + lang('cpp', 'C++', 'native', 'native', ['.cc', '.cpp', '.cxx', '.c++', '.hpp', '.hh', '.hxx', '.ipp']), + lang('zig', 'Zig', 'native', 'native', ['.zig']), + lang('nim', 'Nim', 'native', 'native', ['.nim', '.nims']), + lang('crystal', 'Crystal', 'native', 'native', ['.cr']), + lang('assembly', 'Assembly', 'native', 'native', ['.asm', '.s']), + lang('fortran', 'Fortran', 'native', 'native', ['.f', '.f90', '.f95', '.for']), + lang('verilog', 'Verilog / VHDL', 'native', 'native', ['.sv', '.svh', '.vhd', '.vhdl']), + lang('elixir', 'Elixir', 'beam', 'beam', ['.ex', '.exs']), + lang('erlang', 'Erlang', 'beam', 'beam', ['.erl', '.hrl']), + lang('gleam', 'Gleam', 'beam', 'beam', ['.gleam']), + lang('haskell', 'Haskell', 'functional', 'functional', ['.hs', '.lhs']), + lang('ocaml', 'OCaml', 'functional', 'functional', ['.ml', '.mli']), + lang('elm', 'Elm', 'functional', 'functional', ['.elm']), + lang('purescript', 'PureScript', 'functional', 'functional', ['.purs']), + lang('rescript', 'ReScript', 'functional', 'functional', ['.res', '.resi']), + lang('lisp', 'Lisp', 'functional', 'functional', ['.lisp', '.lsp', '.el']), + lang('scheme', 'Scheme / Racket', 'functional', 'functional', ['.scm', '.ss', '.rkt']), + lang('shell', 'Shell', 'shell', 'shell', ['.sh', '.bash', '.zsh', '.fish', '.ksh']), + lang('powershell', 'PowerShell', 'shell', 'shell', ['.ps1', '.psm1', '.psd1']), + lang('batch', 'Batch', 'shell', 'shell', ['.bat', '.cmd']), + lang('perl', 'Perl', 'shell', 'shell', ['.pl', '.pm']), + lang('vimscript', 'Vim script', 'shell', 'shell', ['.vim']), + lang('lua', 'Lua', 'other', 'other', ['.lua']), + lang('dart', 'Dart', 'mobile', 'other', ['.dart']), + lang('julia', 'Julia', 'data', 'data', ['.jl']), + lang('r', 'R', 'data', 'data', ['.r', '.rmd']), + lang('solidity', 'Solidity', 'other', 'other', ['.sol']), + lang('shaders', 'Shaders', 'gamedev', 'native', ['.glsl', '.frag', '.vert', '.wgsl', '.hlsl', '.metal']), + lang('wat', 'WebAssembly text', 'native', 'native', ['.wat']), + lang('vue', 'Vue', 'web', 'markup', ['.vue']), + lang('svelte', 'Svelte', 'web', 'markup', ['.svelte']), + lang('astro', 'Astro', 'web', 'markup', ['.astro']), + lang('html', 'HTML', 'web', 'markup', ['.html', '.htm', '.xhtml']), + lang('css', 'CSS', 'web', 'markup', ['.css', '.scss', '.sass', '.less', '.styl']), + lang('templates', 'Templates', 'web', 'markup', [ + '.hbs', '.handlebars', '.ejs', '.pug', '.jade', '.liquid', '.njk', '.twig', + '.erb', '.haml', '.slim', '.j2', '.jinja', + ]), + lang('markdown', 'Markdown', 'docs', 'docs', ['.md', '.mdx', '.markdown']), + lang('restructuredtext', 'reStructuredText', 'docs', 'docs', ['.rst']), + lang('asciidoc', 'AsciiDoc', 'docs', 'docs', ['.adoc', '.asciidoc']), + lang('tex', 'TeX', 'docs', 'docs', ['.tex', '.sty', '.cls']), + lang('diagrams', 'Diagrams as code', 'docs', 'docs', ['.mmd', '.puml', '.dot']), + lang('sql', 'SQL', 'data', 'data', ['.sql', '.ddl']), + lang('protobuf', 'Protocol Buffers', 'data', 'data', ['.proto']), + lang('graphql', 'GraphQL', 'data', 'data', ['.graphql', '.gql']), + lang('idl', 'Interface definitions', 'data', 'data', ['.thrift', '.capnp', '.avdl']), + lang('config', 'Config', 'data', 'data', [ + '.json', '.jsonc', '.json5', '.yml', '.yaml', '.toml', '.ini', '.cfg', '.conf', + '.properties', '.xml', '.plist', + ]), + lang('hcl', 'HCL / Terraform', 'infra', 'infra', ['.tf', '.tfvars', '.hcl']), + lang('nix', 'Nix', 'infra', 'infra', ['.nix']), + // Filenames are matched case-insensitively, so `Makefile` covers `makefile`. + // These three also have `tool` twins below: a Dockerfile has lines AND its + // presence is a fact about the project. The language entry owns the lines; the + // tool entry owns the presence. Neither is derived from the other. + lang('makefile', 'Make', 'infra', 'infra', ['.mk'], ['Makefile', 'GNUmakefile']), + lang('cmake', 'CMake', 'infra', 'infra', ['.cmake'], ['CMakeLists.txt']), + lang('dockerfile', 'Dockerfile', 'infra', 'infra', ['.dockerfile'], ['Dockerfile', 'Containerfile']), +]; + +// ── frameworks, SDKs and manifest-detected tooling ──────────────────────────── + +const DEPENDENCY_ENTRIES = [ + // node — UI + dep('react', 'framework', 'React', 'node', ['npm'], ['react', 'react-dom']), + dep('next', 'framework', 'Next.js', 'node', ['npm'], ['next']), + // `vue-framework` / `svelte-framework` / `astro-framework` rather than the bare + // name: ids are unique across the whole registry, and the bare name already + // belongs to the LANGUAGE entry that owns the `.vue` / `.svelte` / `.astro` + // lines. Two entries, two questions — one counts lines, one states presence. + dep('vue-framework', 'framework', 'Vue', 'node', ['npm'], ['vue']), + dep('nuxt', 'framework', 'Nuxt', 'node', ['npm'], ['nuxt']), + dep('svelte-framework', 'framework', 'Svelte', 'node', ['npm'], ['svelte']), + dep('sveltekit', 'framework', 'SvelteKit', 'node', ['npm'], ['@sveltejs/kit']), + dep('angular', 'framework', 'Angular', 'node', ['npm'], ['@angular/core']), + dep('solid', 'framework', 'Solid', 'node', ['npm'], ['solid-js']), + dep('preact', 'framework', 'Preact', 'node', ['npm'], ['preact']), + dep('astro-framework', 'framework', 'Astro', 'node', ['npm'], ['astro']), + dep('remix', 'framework', 'Remix', 'node', ['npm'], [], ['@remix-run/']), + dep('tailwind', 'framework', 'Tailwind CSS', 'node', ['npm'], ['tailwindcss']), + dep('mui', 'framework', 'MUI', 'node', ['npm'], [], ['@mui/']), + dep('styled-components', 'framework', 'styled-components', 'node', ['npm'], + ['styled-components'], ['@emotion/']), + dep('redux', 'framework', 'Redux', 'node', ['npm'], ['redux', '@reduxjs/toolkit']), + dep('zustand', 'framework', 'Zustand', 'node', ['npm'], ['zustand']), + dep('tanstack-query', 'framework', 'TanStack Query', 'node', ['npm'], [], ['@tanstack/']), + dep('d3', 'framework', 'D3', 'node', ['npm'], ['d3']), + dep('three', 'framework', 'three.js', 'node', ['npm'], ['three']), + // node — server + dep('express', 'framework', 'Express', 'node', ['npm'], ['express']), + dep('nest', 'framework', 'NestJS', 'node', ['npm'], ['@nestjs/core']), + dep('fastify', 'framework', 'Fastify', 'node', ['npm'], ['fastify']), + dep('koa', 'framework', 'Koa', 'node', ['npm'], ['koa']), + dep('hono', 'framework', 'Hono', 'node', ['npm'], ['hono']), + dep('socket-io', 'framework', 'Socket.IO', 'node', ['npm'], ['socket.io']), + dep('graphql-js', 'framework', 'GraphQL', 'node', ['npm'], ['graphql'], ['@apollo/']), + dep('prisma', 'framework', 'Prisma', 'node', ['npm'], ['prisma', '@prisma/client']), + dep('drizzle', 'framework', 'Drizzle ORM', 'node', ['npm'], ['drizzle-orm', 'drizzle-kit']), + dep('typeorm', 'framework', 'TypeORM', 'node', ['npm'], ['typeorm']), + dep('sequelize', 'framework', 'Sequelize', 'node', ['npm'], ['sequelize']), + dep('mongoose', 'framework', 'Mongoose', 'node', ['npm'], ['mongoose']), + dep('knex', 'framework', 'Knex', 'node', ['npm'], ['knex', 'kysely']), + // node — desktop / mobile + dep('electron', 'framework', 'Electron', 'node', ['npm'], ['electron']), + dep('tauri', 'framework', 'Tauri', 'node', ['npm', 'cargo'], ['tauri'], ['@tauri-apps/']), + dep('react-native', 'framework', 'React Native', 'node', ['npm'], ['react-native']), + dep('expo', 'framework', 'Expo', 'node', ['npm'], ['expo']), + // node — build / test / lint (tools that happen to live in a manifest) + dep('vite', 'tool', 'Vite', 'node', ['npm'], ['vite']), + dep('webpack', 'tool', 'webpack', 'node', ['npm'], ['webpack']), + dep('rollup', 'tool', 'Rollup', 'node', ['npm'], ['rollup']), + dep('esbuild', 'tool', 'esbuild', 'node', ['npm'], ['esbuild']), + dep('parcel', 'tool', 'Parcel', 'node', ['npm'], ['parcel']), + dep('typescript-compiler', 'tool', 'TypeScript', 'node', ['npm'], ['typescript']), + dep('vitest', 'tool', 'Vitest', 'node', ['npm'], ['vitest']), + dep('jest', 'tool', 'Jest', 'node', ['npm'], ['jest', 'ts-jest']), + dep('mocha', 'tool', 'Mocha', 'node', ['npm'], ['mocha', 'ava']), + dep('playwright', 'tool', 'Playwright', 'node', ['npm'], ['playwright', '@playwright/test']), + dep('cypress', 'tool', 'Cypress', 'node', ['npm'], ['cypress']), + dep('testing-library', 'tool', 'Testing Library', 'node', ['npm'], [], ['@testing-library/']), + dep('eslint', 'tool', 'ESLint', 'node', ['npm'], ['eslint']), + dep('prettier', 'tool', 'Prettier', 'node', ['npm'], ['prettier']), + dep('biome', 'tool', 'Biome', 'node', ['npm'], ['@biomejs/biome']), + dep('turborepo', 'tool', 'Turborepo', 'node', ['npm'], ['turbo', 'nx', 'lerna']), + + // rust + dep('tokio', 'framework', 'Tokio', 'rust', ['cargo'], ['tokio']), + dep('axum', 'framework', 'Axum', 'rust', ['cargo'], ['axum']), + dep('actix', 'framework', 'Actix Web', 'rust', ['cargo'], ['actix-web', 'actix']), + dep('rocket', 'framework', 'Rocket', 'rust', ['cargo'], ['rocket']), + dep('hyper', 'framework', 'Hyper / Tower', 'rust', ['cargo'], ['hyper', 'tower', 'warp']), + dep('serde', 'framework', 'Serde', 'rust', ['cargo'], ['serde', 'serde_json']), + dep('clap', 'framework', 'clap', 'rust', ['cargo'], ['clap']), + dep('tracing', 'framework', 'tracing', 'rust', ['cargo'], ['tracing', 'anyhow', 'thiserror']), + dep('bevy', 'framework', 'Bevy', 'gamedev', ['cargo'], ['bevy']), + dep('wgpu', 'framework', 'wgpu / egui', 'gamedev', ['cargo'], ['wgpu', 'egui']), + dep('sqlx', 'framework', 'SQLx', 'rust', ['cargo'], ['sqlx', 'diesel', 'sea-orm']), + dep('polars', 'framework', 'Polars', 'data', ['cargo', 'python'], ['polars']), + dep('candle', 'framework', 'Candle / Burn', 'ai', ['cargo'], ['candle-core', 'burn']), + dep('wasm-bindgen', 'framework', 'wasm-bindgen', 'rust', ['cargo'], ['wasm-bindgen', 'wasm-pack']), + dep('napi-rs', 'framework', 'napi-rs / PyO3', 'rust', ['cargo'], ['napi', 'pyo3', 'neon']), + dep('rayon', 'framework', 'Rayon', 'rust', ['cargo'], ['rayon']), + dep('criterion', 'tool', 'Criterion', 'rust', ['cargo'], ['criterion']), + + // go + dep('gin', 'framework', 'Gin', 'go', ['gomod'], [], ['github.com/gin-gonic/gin']), + dep('echo', 'framework', 'Echo', 'go', ['gomod'], [], ['github.com/labstack/echo']), + dep('fiber', 'framework', 'Fiber', 'go', ['gomod'], [], ['github.com/gofiber/fiber']), + dep('cobra', 'framework', 'Cobra', 'go', ['gomod'], [], ['github.com/spf13/cobra', 'github.com/spf13/viper']), + dep('grpc-go', 'framework', 'gRPC', 'go', ['gomod'], [], ['google.golang.org/grpc']), + dep('gorm', 'framework', 'GORM', 'go', ['gomod'], [], ['gorm.io/gorm']), + dep('testify', 'tool', 'Testify', 'go', ['gomod'], [], ['github.com/stretchr/testify']), + dep('client-go', 'sdk', 'Kubernetes client-go', 'infra', ['gomod'], [], ['k8s.io/client-go']), + + // python + dep('django', 'framework', 'Django', 'python', ['python'], ['django']), + dep('flask', 'framework', 'Flask', 'python', ['python'], ['flask']), + dep('fastapi', 'framework', 'FastAPI', 'python', ['python'], ['fastapi', 'starlette']), + dep('pydantic', 'framework', 'Pydantic', 'python', ['python'], ['pydantic']), + dep('sqlalchemy', 'framework', 'SQLAlchemy', 'python', ['python'], ['sqlalchemy', 'alembic']), + dep('celery', 'framework', 'Celery', 'python', ['python'], ['celery']), + dep('uvicorn', 'framework', 'Uvicorn / Gunicorn', 'python', ['python'], ['uvicorn', 'gunicorn']), + dep('requests', 'framework', 'requests / httpx', 'python', ['python'], ['requests', 'httpx']), + dep('numpy', 'framework', 'NumPy', 'data', ['python'], ['numpy', 'scipy']), + dep('pandas', 'framework', 'pandas', 'data', ['python'], ['pandas']), + dep('scikit-learn', 'framework', 'scikit-learn', 'ai', ['python'], ['scikit-learn', 'matplotlib']), + dep('pytorch', 'framework', 'PyTorch', 'ai', ['python'], ['torch', 'torchvision']), + dep('tensorflow', 'framework', 'TensorFlow / JAX', 'ai', ['python'], ['tensorflow', 'jax']), + dep('transformers', 'framework', 'Transformers', 'ai', ['python'], ['transformers']), + dep('pytest', 'tool', 'pytest', 'python', ['python'], ['pytest']), + dep('ruff', 'tool', 'Ruff', 'python', ['python'], ['ruff', 'black', 'mypy']), + dep('boto3', 'sdk', 'AWS SDK (boto3)', 'infra', ['python'], ['boto3']), + + // jvm — maven coordinates and gradle strings alike + dep('spring-boot', 'framework', 'Spring Boot', 'jvm', ['jvm'], + ['org.springframework.boot'], ['spring-boot']), + dep('quarkus', 'framework', 'Quarkus', 'jvm', ['jvm'], ['io.quarkus'], ['quarkus-']), + dep('micronaut', 'framework', 'Micronaut', 'jvm', ['jvm'], ['io.micronaut'], ['micronaut-']), + dep('ktor', 'framework', 'Ktor', 'jvm', ['jvm'], ['io.ktor'], ['ktor-']), + dep('hibernate', 'framework', 'Hibernate', 'jvm', ['jvm'], [], ['org.hibernate', 'hibernate-']), + dep('jackson', 'framework', 'Jackson', 'jvm', ['jvm'], [], ['com.fasterxml.jackson', 'jackson-']), + dep('kotlin-coroutines', 'framework', 'Kotlin coroutines', 'jvm', ['jvm'], [], ['kotlinx-coroutines']), + dep('junit', 'tool', 'JUnit', 'jvm', ['jvm'], ['junit', 'org.junit.jupiter'], ['junit-']), + dep('mockito', 'tool', 'Mockito', 'jvm', ['jvm'], ['org.mockito'], ['mockito-']), + + // other ecosystems + dep('flutter', 'framework', 'Flutter', 'mobile', ['pub'], ['flutter']), + dep('riverpod', 'framework', 'Riverpod / Bloc', 'mobile', ['pub'], ['riverpod', 'flutter_bloc', 'bloc']), + dep('dio', 'framework', 'Dio', 'mobile', ['pub'], ['dio']), + dep('phoenix', 'framework', 'Phoenix', 'beam', ['hex'], ['phoenix', 'phoenix_live_view']), + dep('ecto', 'framework', 'Ecto', 'beam', ['hex'], ['ecto', 'ecto_sql', 'plug']), + dep('absinthe', 'framework', 'Absinthe', 'beam', ['hex'], ['absinthe']), + dep('nx', 'framework', 'Nx', 'ai', ['hex'], ['nx', 'axon']), + dep('laravel', 'framework', 'Laravel', 'php', ['composer'], ['laravel/framework']), + dep('symfony', 'framework', 'Symfony', 'php', ['composer'], [], ['symfony/']), + dep('guzzle', 'framework', 'Guzzle', 'php', ['composer'], ['guzzlehttp/guzzle']), + dep('phpunit', 'tool', 'PHPUnit', 'php', ['composer'], ['phpunit/phpunit']), + dep('rails', 'framework', 'Rails', 'ruby', ['rubygems'], ['rails', 'railties']), + dep('sinatra', 'framework', 'Sinatra', 'ruby', ['rubygems'], ['sinatra']), + dep('sidekiq', 'framework', 'Sidekiq / Puma', 'ruby', ['rubygems'], ['sidekiq', 'puma']), + dep('rspec', 'tool', 'RSpec', 'ruby', ['rubygems'], ['rspec', 'rspec-rails', 'rubocop']), + + // AI SDKs — the reason half of these projects exist, so they are named, not + // folded into a generic "http client" bucket. + dep('anthropic-sdk', 'sdk', 'Anthropic SDK', 'ai', ['npm', 'python'], + ['@anthropic-ai/sdk', 'anthropic']), + dep('openai-sdk', 'sdk', 'OpenAI SDK', 'ai', ['npm', 'python'], ['openai']), + dep('google-genai-sdk', 'sdk', 'Google GenAI SDK', 'ai', ['npm', 'python'], + ['@google/generative-ai', 'google-generativeai', '@google/genai']), + dep('langchain', 'sdk', 'LangChain', 'ai', ['npm', 'python'], + ['langchain', 'langgraph'], ['@langchain/', 'langchain-']), + dep('llamaindex', 'sdk', 'LlamaIndex', 'ai', ['npm', 'python'], ['llamaindex', 'llama-index']), + dep('vercel-ai-sdk', 'sdk', 'Vercel AI SDK', 'ai', ['npm'], ['ai'], ['@ai-sdk/']), + dep('mcp-sdk', 'sdk', 'Model Context Protocol', 'ai', ['npm', 'python'], + ['mcp', 'fastmcp'], ['@modelcontextprotocol/']), + dep('ollama', 'sdk', 'Ollama', 'ai', ['npm', 'python'], ['ollama']), + dep('ruflo', 'sdk', 'ruflo / claude-flow', 'ai', ['npm'], + ['claude-flow', 'ruflo', '@claude-flow/cli'], ['@ruvector/']), +]; + +// ── file / directory signatures ─────────────────────────────────────────────── + +const SIGNATURE_ENTRIES = [ + sig('docker', 'Docker', 'infra', { + files: ['dockerfile', 'containerfile', '.dockerignore'], + filePrefixes: ['dockerfile.'], + }), + sig('docker-compose', 'Docker Compose', 'infra', { + filePrefixes: ['docker-compose', 'compose.yml', 'compose.yaml'], + }), + sig('github-actions', 'GitHub Actions', 'infra', { dirs: ['.github/workflows'] }), + sig('gitlab-ci', 'GitLab CI', 'infra', { files: ['.gitlab-ci.yml'] }), + sig('circleci', 'CircleCI', 'infra', { dirs: ['.circleci'] }), + sig('terraform', 'Terraform', 'infra', { + files: ['main.tf', 'versions.tf', 'terraform.tf'], + dirs: ['.terraform'], + }), + sig('kubernetes', 'Kubernetes / Helm', 'infra', { + files: ['chart.yaml', 'kustomization.yaml', 'skaffold.yaml'], + dirs: ['k8s', 'kubernetes', 'charts'], + }), + sig('serverless', 'Serverless / SAM', 'infra', { files: ['serverless.yml', 'template.yaml'] }), + sig('vercel', 'Vercel / Netlify', 'infra', { files: ['vercel.json', 'netlify.toml'] }), + sig('nix-flake', 'Nix', 'infra', { files: ['flake.nix', 'shell.nix', 'default.nix'] }), + sig('devcontainer', 'Dev Container', 'infra', { dirs: ['.devcontainer'] }), + sig('direnv', 'direnv', 'shell', { files: ['.envrc'] }), + sig('make', 'Make', 'infra', { files: ['makefile', 'gnumakefile'] }), + sig('just', 'just', 'infra', { files: ['justfile', '.justfile'] }), + sig('taskfile', 'Task', 'infra', { files: ['taskfile.yml', 'taskfile.yaml'] }), + sig('cmake-build', 'CMake', 'infra', { files: ['cmakelists.txt'] }), + sig('eslint-config', 'ESLint', 'node', { filePrefixes: ['.eslintrc', 'eslint.config.'] }), + sig('prettier-config', 'Prettier', 'node', { filePrefixes: ['.prettierrc', 'prettier.config.'] }), + sig('tsconfig', 'TypeScript config', 'node', { + files: ['tsconfig.json'], filePrefixes: ['tsconfig.'], + }), + sig('pre-commit', 'pre-commit', 'infra', { files: ['.pre-commit-config.yaml'] }), + sig('husky', 'husky / lint-staged', 'node', { dirs: ['.husky'] }), + sig('changesets', 'Changesets', 'node', { dirs: ['.changeset'] }), + sig('editorconfig', 'EditorConfig', 'infra', { files: ['.editorconfig'] }), + sig('dependabot', 'Dependabot / Renovate', 'infra', { + files: ['renovate.json', '.renovaterc', '.github/dependabot.yml'], + }), + sig('claude-code-config', 'Claude Code config', 'ai', { + files: ['claude.md', 'claude.local.md'], dirs: ['.claude'], + }), + sig('agents-md', 'AGENTS.md', 'ai', { files: ['agents.md'] }), + sig('ruflo-state', 'ruflo state', 'ai', { dirs: ['.claude-flow', '.swarm', '.hive-mind'] }), + sig('agentic-qe-state', 'agentic-qe state', 'ai', { dirs: ['.agentic-qe'] }), + sig('codex-config', 'Codex config', 'ai', { dirs: ['.codex'] }), + sig('opencode-config', 'OpenCode config', 'ai', { + files: ['opencode.json', 'opencode.jsonc'], dirs: ['.opencode'], + }), +]; + +// ── the deliberate non-source list ──────────────────────────────────────────── +// +// Extensions we have LOOKED AT and decided hold bytes, not lines. This is what +// separates a decision from an oversight: an extension here is a stated exclusion +// the surfaces can name, while an extension in neither list is an unrecognized +// tail row — a to-do the next release can close. Without this split the tail fills +// with .png and .sqlite and stops being a to-do list at all. +// +// `.svg` is the judgement call in this list. Hand-authored SVG really is source, +// but most SVG on a machine is exported by a tool, and letting generated art into +// a line count would quietly inflate every project that ships icons. Recorded here +// as a reviewable decision rather than left to chance. +export const NON_SOURCE_EXTENSIONS = Object.freeze([ + // images, media, fonts + '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico', '.icns', '.tif', '.tiff', + '.svg', '.psd', '.ai', '.mp4', '.mov', '.webm', '.avi', '.mkv', '.mp3', '.wav', + '.ogg', '.flac', '.woff', '.woff2', '.ttf', '.otf', '.eot', + // archives and packages + '.zip', '.tar', '.gz', '.tgz', '.bz2', '.xz', '.zst', '.7z', '.rar', '.dmg', '.iso', + '.deb', '.rpm', '.apk', '.ipa', '.pkg', '.whl', '.crate', '.nupkg', + // compiled and linked artefacts + '.exe', '.dll', '.so', '.dylib', '.a', '.o', '.obj', '.lib', '.bin', '.wasm', + '.class', '.jar', '.war', '.ear', '.pyc', '.pyo', '.pyd', '.node', '.rlib', + '.rmeta', '.pdb', '.d', '.map', + // stores, indexes and model weights + '.db', '.db-wal', '.db-shm', '.sqlite', '.sqlite3', '.sqlite-wal', '.sqlite-shm', + '.mdb', '.mmdb', '.rvf', '.hnsw', '.idx', '.index', '.pack', '.parquet', '.arrow', + '.feather', '.npy', '.npz', '.pt', '.pth', '.onnx', '.safetensors', '.gguf', '.ggml', + // records, dumps and scratch + '.log', '.jsonl', '.ndjson', '.csv', '.tsv', '.txt', '.pdf', '.docx', '.xlsx', + '.pptx', '.lock', '.bak', '.backup', '.tmp', '.swp', '.part', '.pid', '.sock', + '.pem', '.key', '.crt', '.cer', '.p12', '.der', + // tool output that happens to be text: a report is not a line someone wrote + '.sarif', '.excalidraw', '.snap', +]); + +const nonSource = new Set(NON_SOURCE_EXTENSIONS); + +/** Is this extension a stated non-source exclusion (as opposed to something the + * registry has simply never been taught)? The distinction is the whole point of + * the unrecognized tail. */ +export function isNonSourceExtension(ext) { + return nonSource.has(String(ext ?? '').toLowerCase()); +} + +/** The whole catalog, frozen. Order is language → dependency → signature so a + * consumer that renders the registry itself gets a stable reading order. */ +export const STACK_REGISTRY = Object.freeze([ + ...LANGUAGE_ENTRIES, ...DEPENDENCY_ENTRIES, ...SIGNATURE_ENTRIES, +]); + +// ── indices ─────────────────────────────────────────────────────────────────── +// +// Built once at import from the frozen data above — pure computation, no I/O. +// Every index is FIRST-WINS on a collision and records what it dropped in +// REGISTRY_CONFLICTS, so an accidental duplicate is a visible, testable fact +// rather than a silently shadowed entry. + +/** Collisions found while indexing. Empty in a healthy registry; a unit test + * asserts that, which is what keeps this a curated artifact rather than a pile. */ +export const REGISTRY_CONFLICTS = []; + +const byId = new Map(); +const byExtension = new Map(); +const byFilename = new Map(); +const byDependency = new Map(); // `${manifest}${name}` and prefix list per manifest +const prefixesByManifest = new Map(); + +const claim = (map, key, entry, scope) => { + const held = map.get(key); + if (held) { + REGISTRY_CONFLICTS.push({ scope, key, keptId: held.id, droppedId: entry.id }); + return; + } + map.set(key, entry); +}; + +for (const entry of STACK_REGISTRY) { + claim(byId, entry.id, entry, 'id'); + if (entry.match.by === 'extension') { + for (const ext of entry.match.extensions) claim(byExtension, ext, entry, 'extension'); + for (const name of entry.match.filenames) claim(byFilename, name, entry, 'filename'); + } else if (entry.match.by === 'dependency') { + for (const manifest of entry.match.manifests) { + for (const name of entry.match.names) { + claim(byDependency, `${manifest}${name}`, entry, 'dependency'); + } + if (entry.match.prefixes.length) { + if (!prefixesByManifest.has(manifest)) prefixesByManifest.set(manifest, []); + for (const prefix of entry.match.prefixes) { + prefixesByManifest.get(manifest).push({ prefix, entry }); + } + } + } + } +} +// Longest prefix first, so `@langchain/` never loses to a shorter neighbour. +for (const list of prefixesByManifest.values()) list.sort((a, b) => b.prefix.length - a.prefix.length); +// An extension cannot be both a counted language and a stated non-source +// exclusion; whichever way that got decided it was decided twice. +for (const ext of NON_SOURCE_EXTENSIONS) { + const held = byExtension.get(ext); + if (held) REGISTRY_CONFLICTS.push({ scope: 'non-source', key: ext, keptId: held.id, droppedId: 'non-source' }); +} +Object.freeze(REGISTRY_CONFLICTS); + +// ── accessors ───────────────────────────────────────────────────────────────── + +/** Every entry, or every entry of one kind. Returns a fresh array; the entries + * themselves are frozen. */ +export function stackEntries(kind = null) { + return kind ? STACK_REGISTRY.filter((entry) => entry.kind === kind) : [...STACK_REGISTRY]; +} + +/** One entry by id, or null. Ids are globally unique across kinds. */ +export function stackEntryById(id) { + return byId.get(String(id ?? '')) ?? null; +} + +/** The language that owns a file extension (leading dot, any case), or null. + * Null is the answer that matters: it means "not counted", not "zero lines". */ +export function languageForExtension(ext) { + return byExtension.get(String(ext ?? '').toLowerCase()) ?? null; +} + +/** The language that owns a bare filename (`Makefile`, `Dockerfile`), or null. */ +export function languageForFilename(name) { + return byFilename.get(String(name ?? '').toLowerCase()) ?? null; +} + +/** The manifest family a filename belongs to, or null. `requirements-dev.txt` and + * friends are matched by shape because the suffix is convention, not spec. */ +export function manifestKindFor(filename) { + const name = String(filename ?? '').toLowerCase(); + const exact = MANIFEST_FILES[name]; + if (exact) return exact; + return /^requirements[\w.-]*\.txt$/.test(name) ? 'python' : null; +} + +/** The registry entry a dependency name resolves to inside one manifest family, + * or null — which is exactly what makes it part of the unrecognized tail. */ +export function dependencyEntry(manifestKind, depName) { + const name = String(depName ?? '').toLowerCase(); + if (!name) return null; + const exact = byDependency.get(`${manifestKind}${name}`); + if (exact) return exact; + for (const { prefix, entry } of prefixesByManifest.get(manifestKind) ?? []) { + if (name.startsWith(prefix)) return entry; + } + return null; +} + +/** + * Every signature entry, for the caller that matches collected paths in one pass. + * + * @returns {Array<{ id: string, kind: string, name: string, ecosystem: string, + * match: { by: string, files: string[], dirs: string[], filePrefixes: string[] } }>} + */ +export function signatureEntries() { + return /** @type {any} */ (STACK_REGISTRY.filter((entry) => entry.match.by === 'signature')); +} + +/** The one-line provenance a panel prints next to a changed number: which + * registry produced it and how wide that registry is. */ +export function registryStats() { + const counts = { language: 0, framework: 0, sdk: 0, tool: 0 }; + for (const entry of STACK_REGISTRY) counts[entry.kind] += 1; + return Object.freeze({ + version: STACK_REGISTRY_VERSION, + entries: STACK_REGISTRY.length, + languages: counts.language, + frameworks: counts.framework, + sdks: counts.sdk, + tools: counts.tool, + extensions: byExtension.size, + filenames: byFilename.size, + nonSourceExtensions: NON_SOURCE_EXTENSIONS.length, + manifestKinds: MANIFEST_KINDS.length, + }); +} diff --git a/src/lib/footprint/storage.mjs b/src/lib/footprint/storage.mjs index 7166a18..fd4ce68 100644 --- a/src/lib/footprint/storage.mjs +++ b/src/lib/footprint/storage.mjs @@ -10,6 +10,26 @@ // runs. npx.mjs's pruneNpxStale is deliberately NOT imported; only its // read-only scanNpxStale is. // +// SAFETY IS A FIELD, NOT A TONE OF VOICE. Every candidate carries `safety`: +// 'regenerable' (the owning tool refetches it on demand — the npm content cache, +// the Homebrew download cache, the brain's superseded KB copies) or 'review' +// (plausible but NOT safe to state as removable — mise has eight node entries on +// this machine and some of them are the aliases a live toolchain resolves +// through; a browser revision may still be pinned by an installed package). +// A review row is a pointer at something to look at, never a figure to sweep, +// and `summarizeReclaimables` totals the two tiers SEPARATELY on purpose: a +// combined "you could free N" that mixes them would be the honest-measurement +// contract broken at the last mile. +// +// `bytesMeaning` says what the bytes on a row are: 'candidate' (the bytes the +// row is actually about) or 'installed' (what is on disk, offered as context on +// a review row that has no defensible candidate subset — the mise rows). +// +// Absent candidate = absent. A cache root that does not exist, a family with no +// superseded members, a walk that measured a real zero: none of those produce a +// row. Only something that IS there is listed, so an empty advisory panel means +// "nothing crossed a threshold" rather than "nothing was looked at". +// // Metadata only (invariant 1). Every figure comes from dirents, lstat sizes and // mtimes. A transcript's contents are never opened — which is also why codex // transcripts have no project attribution here: codex rollout PATHS are dated, @@ -24,12 +44,13 @@ // and skipped entirely when `detectWorktrees` is false. import fs from 'node:fs'; import path from 'node:path'; -import { home, claudeDir, codexDir, configDir } from '../paths.mjs'; +import { home, claudeDir, codexDir, configDir, isWindows } from '../paths.mjs'; import { defaultOpencodeDbPath } from '../usage-opencode.mjs'; import { scanNpxStale } from '../npx.mjs'; import { npxEnvNodes } from './install.mjs'; +import { decodeClaudeProjectDir } from './project-sources.mjs'; import { - walkTree, rootMeasurements, measured, unknown, sumMeasurements, statNode, + walkTree, rootMeasurements, measured, unknown, sumMeasurements, statNode, hasValue, } from './walk.mjs'; export const STORAGE_CATEGORIES = Object.freeze([ @@ -47,9 +68,28 @@ export const STORAGE_DEFAULTS = Object.freeze({ npxEnvIdleDays: 90, worktreeIdleDays: 90, maxWorktreeWalks: 32, + // How many members of one family (dated KB copies, browser revisions, runtime + // versions) are walked. A family that exceeds it reports what it measured as a + // floor rather than walking an unbounded number of trees. + maxFamilyWalks: 64, samplePaths: 5, }); +/** The two safety tiers, in the order a panel should present them. Two and not + * three: a "definitely dead" tier would be a claim this module cannot + * substantiate from directory metadata alone. */ +export const RECLAIM_SAFETY_TIERS = Object.freeze(['regenerable', 'review']); + +/** What each tier promises, carried in the payload so no surface has to invent + * the wording — and so the difference between the two is impossible to render + * as the same thing. */ +export const RECLAIM_SAFETY_MEANING = Object.freeze({ + regenerable: 'The owning tool refetches this on demand. Removing it costs download time, ' + + 'not data.', + review: 'Plausible but not safe to call removable: some of these may be in use. Review them ' + + 'individually — this is not a total to sweep.', +}); + /** Host ledger and log files that are named by generation * (`state_5.sqlite`, `logs_2.sqlite`, plus their -wal/-shm siblings). Matching * the family rather than one name is deliberate: codex bumps the generation @@ -283,10 +323,19 @@ function leafKeysFor(root, file) { /** * The storage section of a FootprintSnapshot. * + * `consumers` is an already-collected ranked-consumers payload (consumers.mjs). + * When supplied, a detector that needs the size of a path that view already + * walked adopts that figure instead of walking the tree a second time — the npm + * content cache alone is ~10^5 files, and measuring it twice in one deep scan is + * pure I/O cost for an identical answer. + * * @param {{ * projects?: string[]|null, now?: () => number, walk?: typeof walkTree, * roots?: StorageRoot[]|null, limits?: object, growthDays?: number, topN?: number, * maxChildren?: number, reclaim?: object, detectWorktrees?: boolean, + * detectCaches?: boolean, detectOrphanedTranscripts?: boolean, + * consumers?: object|null, env?: NodeJS.ProcessEnv, + * decodeDir?: typeof decodeClaudeProjectDir, * fsImpl?: typeof fs, * }} [options] */ @@ -301,6 +350,11 @@ export function collectStorage({ maxChildren = STORAGE_DEFAULTS.maxChildren, reclaim = {}, detectWorktrees = true, + detectCaches = true, + detectOrphanedTranscripts = true, + consumers = null, + env = process.env, + decodeDir = decodeClaudeProjectDir, fsImpl = fs, } = {}) { const asOf = now(); @@ -314,6 +368,10 @@ export function collectStorage({ const sessionLeaves = []; const files = []; const agedTranscripts = new Map(); + // Per transcript-project-directory totals, kept alongside the tree because the + // orphaned-transcript detector needs exactly what this walk already measured — + // re-walking those directories to learn the same bytes would be waste. + const transcriptProjects = new Map(); let anyDegraded = false; const trimTop = (list) => { @@ -362,6 +420,20 @@ export function collectStorage({ attribution, }); bump(leafParent, bytes, mtimeMs); + if (project && root.category === 'transcripts') { + const key = `${root.id}${project}`; + const acc = transcriptProjects.get(key) ?? { + host: root.host, rootPath: root.path, dir: project, + path: path.join(root.path, project), + bytes: 0, files: 0, newestMtimeMs: null, + }; + acc.bytes += bytes; + acc.files += 1; + if (acc.newestMtimeMs === null || mtimeMs > acc.newestMtimeMs) { + acc.newestMtimeMs = mtimeMs; + } + transcriptProjects.set(key, acc); + } } else if (root.layout === 'flat-sessions') { rootNode.attribution = 'none'; } @@ -444,6 +516,11 @@ export function collectStorage({ files.sort((a, b) => b.bytes - a.bytes); sessionLeaves.sort((a, b) => b.bytes - a.bytes); + const reclaimables = collectReclaimables({ + asOf, agedTranscripts, transcriptProjects, projects, opts, walk, limits, + detectWorktrees, detectCaches, detectOrphanedTranscripts, consumers, env, decodeDir, fsImpl, + }); + return { asOf, categories: tree, @@ -454,9 +531,8 @@ export function collectStorage({ growth: buildGrowth(growth, { asOf, growthDays }), topSessions: sessionLeaves.slice(0, topN), topFiles: files.slice(0, topN), - reclaimables: collectReclaimables({ - asOf, agedTranscripts, projects, opts, walk, limits, detectWorktrees, fsImpl, - }), + reclaimables, + reclaimSummary: summarizeReclaimables(reclaimables, { asOf }), complete: !anyDegraded, }; } @@ -491,38 +567,136 @@ export function buildGrowth(growthByHost, { asOf, growthDays }) { }; } -/** Advisory rows only — see this module's header. Nothing here removes - * anything; `cleanupHint` names the CLI that already owns the removal. */ +/** + * Advisory rows only — see this module's header. Nothing here removes anything; + * `cleanupHint` names the CLI that already owns the removal. + * + * @typedef {{ value: number|null, status: string, reason: string|null, + * asOf: number|null, partial: boolean }} Measurement + * @typedef {{ + * id: string, kind: string, label: string, path: string, samplePaths: string[], + * matchedCount: number|null, bytes: Measurement, files: Measurement, + * safety: 'regenerable'|'review', bytesMeaning: 'candidate'|'installed', + * keeps: Array<{ path: string, label: string, bytes: Measurement }>, + * rationale: string, cleanupHint: string|null, advisory: true, + * }} ReclaimableCandidate + */ export function collectReclaimables({ - asOf, agedTranscripts, projects, opts, walk, limits, detectWorktrees, fsImpl, + asOf, agedTranscripts, transcriptProjects = new Map(), projects, opts, walk, limits, + detectWorktrees, detectCaches = true, detectOrphanedTranscripts = true, + consumers = null, env = process.env, decodeDir = decodeClaudeProjectDir, fsImpl, }) { const rows = []; const days = (ms) => Math.floor((asOf - ms) / 86_400_000); + const ctx = { + asOf, opts, walk, limits, fsImpl, adopt: adoptedConsumerFigures(consumers), + }; for (const acc of agedTranscripts.values()) { - rows.push({ + rows.push(candidate({ id: `aged-transcripts:${acc.host}`, kind: 'aged-transcripts', label: `${acc.host} transcripts older than ${opts.transcriptAgeDays}d`, path: acc.root, samplePaths: acc.samples, + matchedCount: acc.files, bytes: measured(acc.bytes, { asOf }), files: measured(acc.files, { asOf }), + // Not regenerable in any sense: a transcript is the only copy of the + // session it records, and Historical usage is denominated in them. + safety: 'review', rationale: `${acc.files} file(s) untouched for ${opts.transcriptAgeDays}d or more; ` + `oldest ${days(acc.oldestMtimeMs)}d. Historical usage reads these — removing them ` + 'removes that history too.', cleanupHint: null, - advisory: true, - }); + })); } rows.push(...npxReclaimables({ asOf, opts, walk, limits, fsImpl })); + if (detectOrphanedTranscripts) { + rows.push(...orphanedTranscriptReclaimables({ + asOf, opts, transcriptProjects, decodeDir, fsImpl, + })); + } + if (detectCaches) { + rows.push(...supersededSnapshotReclaimables(ctx, snapshotFamilies({ env }))); + rows.push(...regenerableCacheReclaimables(ctx, regenerableCacheRoots({ env }))); + rows.push(...browserRevisionReclaimables(ctx, browserRevisionRoots({ env }))); + rows.push(...runtimeVersionReclaimables(ctx, runtimeVersionRoots({ env }))); + } if (detectWorktrees && Array.isArray(projects)) { rows.push(...worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImpl })); } return rows.sort((a, b) => (b.bytes.value ?? 0) - (a.bytes.value ?? 0)); } +/** Do any two of these rows describe the same path, or one inside another? Two + * such rows cover some of the same bytes — an aged transcript can also sit in a + * project that no longer exists — and adding them would report space twice. */ +function rowsOverlap(rows) { + const keys = rows.map((row) => (row.path ? pathKey(row.path) : null)).filter(Boolean); + for (let i = 0; i < keys.length; i++) { + for (let j = i + 1; j < keys.length; j++) { + if (keys[i] === keys[j]) return true; + const rel = path.relative(keys[i], keys[j]); + const nested = rel && !rel.startsWith('..') && !path.isAbsolute(rel); + const inverse = path.relative(keys[j], keys[i]); + if (nested || (inverse && !inverse.startsWith('..') && !path.isAbsolute(inverse))) return true; + } + } + return false; +} + +/** + * Per-tier totals, and deliberately NO combined figure. Summing a regenerable + * cache with a runtime tree that may be live would produce the one number a + * reader would act on and the one number this module cannot stand behind. + * + * A tier whose own rows overlap reports its total as unknown-with-reason rather + * than as a sum that counts the same bytes twice — the rowCount still stands, + * and each row still carries its own measured figure. + */ +export function summarizeReclaimables(rows, { asOf = null } = {}) { + const list = Array.isArray(rows) ? rows : []; + return { + tiers: RECLAIM_SAFETY_TIERS.map((safety) => { + const tier = list.filter((row) => row.safety === safety); + // Only 'candidate' bytes are summable: an 'installed' figure is context on + // a review row, not space the row claims is available. + const members = tier.filter((row) => row.bytesMeaning === 'candidate'); + const overlapping = rowsOverlap(members); + return { + safety, + meaning: RECLAIM_SAFETY_MEANING[safety], + rowCount: tier.length, + bytes: overlapping + ? unknown('rows in this tier describe overlapping paths, so a sum would count the ' + + 'same bytes twice') + : sumMeasurements(members.map((row) => row.bytes), { asOf }), + summedRows: overlapping ? 0 : members.length, + contextOnlyRows: tier.length - members.length, + }; + }), + combined: null, + combinedNote: 'The tiers are reported separately and never added: only the regenerable ' + + 'total is space a tool would rebuild by itself.', + }; +} + +/** Fill in the fields every row carries, so no detector can ship a candidate + * without a safety tier or a statement of what its bytes mean. */ +function candidate(row) { + return { + samplePaths: [], + matchedCount: null, + keeps: [], + bytesMeaning: 'candidate', + cleanupHint: null, + ...row, + advisory: true, + }; +} + /** Stale npx cache envs. Two independent rationales, both read-only: a cached * copy strictly older than its installed global baseline (npx.mjs's version * verdict — the bug that kept a machine running a retired ruflo), and an env @@ -546,19 +720,610 @@ export function npxReclaimables({ asOf, opts, walk, limits, fsImpl }) { + `older than installed ${stale.map((s) => s.installed).join(', ')}`); } if (idle) why.push(`untouched for ${Math.floor((asOf - env.newestMtimeMs) / 86_400_000)}d`); - rows.push({ + rows.push(candidate({ id: `stale-npx-env:${env.id}`, kind: 'stale-npx-env', label: `npx cache env (${env.packages.join(', ') || 'unkeyed'})`, path: env.path, - samplePaths: [], bytes: env.bytes, files: env.files, + safety: 'regenerable', rationale: `${why.join('; ')}. npx re-fetches on demand, so the cache is reproducible.`, cleanupHint: 'ak sync prunes version-stale envs (npx.pruneNpxStale)', - advisory: true, + })); + } + return rows; +} + +// ── shared detector plumbing ────────────────────────────────────────────────── + +// Third-party cache conventions are spelled out here rather than in paths.mjs +// for the reason consumers.mjs states: that module owns the kit's own path +// contract, and fifty foreign tools' cache layouts would make it harder to +// audit. Platform variants are listed side by side instead of switched on +// process.platform, so the wrong-platform root simply reads absent and a machine +// carrying both (a tool that moved its cache) reports both. +const xdgCache = (env) => env.XDG_CACHE_HOME || path.join(home, '.cache'); +const xdgData = (env) => env.XDG_DATA_HOME || path.join(home, '.local', 'share'); +const macCache = () => path.join(home, 'Library', 'Caches'); +const winLocalAppData = (env) => env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'); + +// APFS and NTFS are case-insensitive, so an adopted figure keyed by an +// exact-case path would silently miss on macOS and Windows. +const foldCase = process.platform !== 'linux'; +const pathKey = (target) => { + const abs = path.resolve(target); + return foldCase ? abs.toLowerCase() : abs; +}; + +/** + * A lookup from absolute path to a figure the ranked-consumers view already + * measured, or a function that always misses when no such view was supplied. + * Exact paths only: a consumers row for a glob FAMILY carries one total for the + * whole family and cannot answer for an individual member. + * + * @param {{ rows?: any[] }|null} consumers a collectConsumers payload + * @returns {(target: string) => ({ presence: string, bytes: Measurement, + * files: Measurement, newestMtimeMs: number|null })|null} + */ +export function adoptedConsumerFigures(consumers) { + const index = new Map(); + for (const row of consumers?.rows ?? []) { + if (!row?.path || row.residual || row.presence !== 'present' || !hasValue(row.bytes)) continue; + index.set(pathKey(row.path), { + presence: 'present', + bytes: row.bytes, + files: row.files ?? unknown('file count not carried by the adopted figure'), + newestMtimeMs: row.newestMtimeMs ?? null, }); } + return (target) => (target ? index.get(pathKey(target)) ?? null : null); +} + +/** One node's figures: the already-measured answer when the consumers view has + * one for this exact path, otherwise one bounded walk. */ +function measureNode(target, ctx) { + const adopted = ctx.adopt?.(target); + if (adopted) return adopted; + const result = ctx.walk(target, { ...ctx.limits, fsImpl: ctx.fsImpl }); + return { + ...rootMeasurements(result, { asOf: ctx.asOf }), + newestMtimeMs: result.newestMtimeMs ?? null, + }; +} + +/** Immediate entries of a directory. ENOENT is an ABSENCE — that tool is not + * installed here, which is not a failed measurement — and every other errno is + * a degradation the caller must report rather than swallow. Symlinked entries + * are classified but never followed or measured. */ +function listMembers(dir, fsImpl) { + try { + return { status: 'ok', reason: null, entries: fsImpl.readdirSync(dir, { withFileTypes: true }) }; + } catch (err) { + const code = err?.code || 'io'; + return { status: code === 'ENOENT' ? 'absent' : 'degraded', reason: code, entries: [] }; + } +} + +/** A root that exists but could not be listed. Reported rather than dropped: + * "there is nothing to reclaim here" and "we could not look" are different + * answers, and only one of them is a measurement (invariant 2). */ +function unreadableCandidate({ id, kind, label, target, reason, safety, cleanupHint = null }) { + return candidate({ + id: `${id}:unreadable`, + kind, + label: `${label} (unreadable)`, + path: target, + bytes: unknown(reason), + files: unknown(reason), + safety, + rationale: `${target} could not be listed (${reason}), so whether anything here is ` + + 'reclaimable is unknown rather than none.', + cleanupHint, + }); +} + +/** Walk a family's members under a walk budget. A cap makes the sum a floor and + * says so through `partial`, which is what "≥ N" renders from; a budget already + * spent before this family was reached yields unknown, because zero members + * measured is not a measurement of zero bytes. */ +function measureMembers(members, ctx, limit = ctx.opts.maxFamilyWalks) { + const walked = members.slice(0, Math.max(0, limit)) + .map((member) => ({ ...member, ...measureNode(member.path, ctx) })); + const capped = members.length > walked.length; + if (members.length && !walked.length) { + const reason = 'the walk budget for this root was spent before this node was reached'; + return { walked, capped, bytes: unknown(reason), files: unknown(reason) }; + } + const bytes = sumMeasurements(walked.map((m) => m.bytes), { asOf: ctx.asOf }); + const files = sumMeasurements(walked.map((m) => m.files), { asOf: ctx.asOf }); + return { + walked, + capped, + bytes: capped && hasValue(bytes) ? { ...bytes, partial: true } : bytes, + files: capped && hasValue(files) ? { ...files, partial: true } : files, + }; +} + +/** Is there anything to advise about? A measured zero is a real zero, and a real + * zero is not a candidate — an "0 B reclaimable" row is an unknown wearing a + * number. An unmeasured figure still earns its row, because not knowing is + * itself the finding. */ +const worthListing = (bytes) => !hasValue(bytes) || bytes.value > 0; + +// ── superseded snapshot copies ──────────────────────────────────────────────── + +/** + * Families of dated copies an installer leaves beside the copy in use. The + * RuvNet Brain is the one on this machine and the largest safe win on it: five + * `kb.bak-` directories totalling ~11 GB beside a 1.9 GB active `kb/`. + * + * @param {{ env?: NodeJS.ProcessEnv }} [options] + */ +export function snapshotFamilies({ env = process.env } = {}) { + const brain = path.join(xdgCache(env), 'ruvnet-brain'); + return [{ + id: 'brain-kb-snapshots', + label: 'RuvNet Brain superseded KB copies', + dir: brain, + // `kb.bak` cannot match `kb`, so the active KB can never be enumerated as + // one of its own backups. + prefix: 'kb.bak', + active: path.join(brain, 'kb'), + activeLabel: 'active KB (kb/)', + what: 'The brain installer copies the knowledge base aside before each update and never ' + + 'removes the copy, so one accumulates per update.', + reproducible: 'A knowledge base is rebuilt by re-running the installer ' + + '(npx ruvnet-brain --doctor).', + cleanupHint: 'remove the dated kb.bak-* directories (npx ruvnet-brain --doctor rebuilds)', + }]; +} + +/** The `YYYY-MM-DD` a dated copy names, when it names one. Used only for the + * rationale's range; a member whose name carries no date is still counted. */ +const datePart = (name) => name.match(/(\d{4}-\d{2}-\d{2})/)?.[1] ?? null; + +/** + * Dated, superseded copies beside an active one — the single largest safe win + * measured on this machine. The active copy is measured too and reported in + * `keeps`, never inside the candidate figure: the row's whole credibility is + * that it can say what it is NOT proposing to touch. + */ +export function supersededSnapshotReclaimables(ctx, families) { + const rows = []; + for (const family of families ?? []) { + const listing = listMembers(family.dir, ctx.fsImpl); + if (listing.status === 'absent') continue; + if (listing.status === 'degraded') { + rows.push(unreadableCandidate({ + id: family.id, kind: 'superseded-snapshots', label: family.label, + target: family.dir, reason: listing.reason, safety: 'regenerable', + cleanupHint: family.cleanupHint, + })); + continue; + } + const members = listing.entries + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink() + && entry.name.startsWith(family.prefix)) + .map((entry) => ({ name: entry.name, path: path.join(family.dir, entry.name) })) + .sort((a, b) => a.name.localeCompare(b.name)); + if (!members.length) continue; + + const { walked, capped, bytes, files } = measureMembers(members, ctx); + if (!worthListing(bytes)) continue; + const dates = members.map((m) => datePart(m.name)).filter(Boolean); + const span = dates.length >= 2 ? ` (${dates[0]} through ${dates[dates.length - 1]})` + : (dates.length === 1 ? ` (${dates[0]})` : ''); + const active = measureNode(family.active, ctx); + rows.push(candidate({ + id: family.id, + kind: 'superseded-snapshots', + label: `${members.length} superseded copies of ${path.basename(family.active)}`, + path: family.dir, + samplePaths: walked.slice(0, ctx.opts.samplePaths).map((m) => m.path), + matchedCount: members.length, + bytes, + files, + safety: 'regenerable', + keeps: [{ + path: family.active, + label: family.activeLabel, + bytes: active.presence === 'absent' + ? unknown('the active copy is not on disk') + : active.bytes, + }], + rationale: `${members.length} dated copies${span}${capped ? ', of which only ' + + `${walked.length} were measured` : ''}. ${family.what} Nothing reads them: ` + + `the ${family.activeLabel} is measured separately, is excluded from this figure, and ` + + `is not a candidate. ${family.reproducible}`, + cleanupHint: family.cleanupHint, + })); + } + return rows; +} + +// ── regenerable caches ──────────────────────────────────────────────────────── + +/** + * Whole cache roots whose owner refetches them on demand. These are the biggest + * genuinely-safe rows on a developer machine — the npm content-addressable cache + * alone measures ~22 GB here — and they are invisible in the per-host storage + * tree because they belong to no host. + * + * @param {{ env?: NodeJS.ProcessEnv }} [options] + */ +export function regenerableCacheRoots({ env = process.env } = {}) { + const cache = xdgCache(env); + const roots = [ + { + id: 'npm-cacache', + kind: 'regenerable-cache', + label: 'npm content-addressable cache', + path: path.join(home, '.npm', '_cacache'), + what: 'Every package tarball and registry response npm has downloaded, keyed by content ' + + 'hash. It grows monotonically: npm adds to it and never prunes it.', + cleanupHint: 'npm cache clean --force', + }, + { + id: 'homebrew-downloads', + kind: 'regenerable-cache', + label: 'Homebrew download cache', + path: path.join(macCache(), 'Homebrew'), + what: 'Bottles, source tarballs and the formula API responses brew downloaded, kept after ' + + 'the install they were for.', + cleanupHint: 'brew cleanup', + }, + { + id: 'homebrew-downloads-xdg', + kind: 'regenerable-cache', + label: 'Homebrew download cache', + path: path.join(cache, 'Homebrew'), + what: 'Bottles, source tarballs and the formula API responses brew downloaded, kept after ' + + 'the install they were for.', + cleanupHint: 'brew cleanup', + }, + ]; + if (isWindows) { + roots.push({ + id: 'npm-cacache-win', + kind: 'regenerable-cache', + label: 'npm content-addressable cache', + path: path.join(winLocalAppData(env), 'npm-cache', '_cacache'), + what: 'Every package tarball and registry response npm has downloaded, keyed by content ' + + 'hash. It grows monotonically: npm adds to it and never prunes it.', + cleanupHint: 'npm cache clean --force', + }); + } + return roots; +} + +/** One row per present cache root. An absent root produces nothing at all — + * a tool that is not installed is not a reclaimable zero. */ +export function regenerableCacheReclaimables(ctx, roots) { + const rows = []; + for (const root of roots ?? []) { + const node = measureNode(root.path, ctx); + if (node.presence === 'absent') continue; + if (!worthListing(node.bytes)) continue; + rows.push(candidate({ + id: root.id, + kind: root.kind, + label: root.label, + path: root.path, + bytes: node.bytes, + files: node.files, + safety: 'regenerable', + rationale: `${root.what} Nothing here is unique: the tool refetches what it needs on the ` + + 'next install, so the cost of clearing it is download time, not data.', + cleanupHint: root.cleanupHint, + })); + } + return rows; +} + +// ── superseded browser downloads ────────────────────────────────────────────── + +/** + * Roots where a browser installer keeps one directory per revision and adds a + * new one on every version bump, never removing the old. `depth` is where the + * revision directories live: Playwright keeps them at the root + * (`chromium-1223`), Puppeteer one level down (`chrome/mac_arm-149.0.7827.22`). + * + * @param {{ env?: NodeJS.ProcessEnv }} [options] + */ +export function browserRevisionRoots({ env = process.env } = {}) { + const cache = xdgCache(env); + const playwright = { + kind: 'superseded-browser-revisions', + label: 'Playwright browser builds', + depth: 1, + installer: 'npx playwright install', + }; + return [ + { ...playwright, id: 'playwright-mac', path: path.join(macCache(), 'ms-playwright') }, + { ...playwright, id: 'playwright-xdg', path: path.join(cache, 'ms-playwright') }, + { ...playwright, id: 'playwright-win', path: path.join(winLocalAppData(env), 'ms-playwright') }, + { + kind: 'superseded-browser-revisions', + id: 'puppeteer', + label: 'Puppeteer browser builds', + path: path.join(cache, 'puppeteer'), + depth: 2, + installer: 'npx puppeteer browsers install', + }, + ]; +} + +/** Split `chromium_headless_shell-1223` into its family and its revision. The + * revision must contain a digit, which is what keeps Playwright's + * `mcp-chrome-profile` — a browser PROFILE, not a revision — out of the + * families entirely. */ +function splitRevision(name) { + const cut = name.lastIndexOf('-'); + if (cut <= 0 || cut === name.length - 1) return null; + const revision = name.slice(cut + 1); + if (!/\d/.test(revision)) return null; + return { family: name.slice(0, cut), revision }; +} + +/** Revision directories under a root, at the root itself (`depth` 1) or one + * level down (`depth` 2). Never recursive: a browser build's own contents are + * not revisions. */ +function revisionMembers(root, ctx) { + const listing = listMembers(root.path, ctx.fsImpl); + if (listing.status !== 'ok') return { status: listing.status, reason: listing.reason, members: [] }; + const holders = root.depth === 2 + ? listing.entries.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => ({ dir: path.join(root.path, entry.name), scope: entry.name })) + : [{ dir: root.path, scope: '' }]; + const members = []; + for (const holder of holders) { + const inner = root.depth === 2 ? listMembers(holder.dir, ctx.fsImpl) : listing; + if (inner.status !== 'ok') continue; + for (const entry of inner.entries) { + if (!entry.isDirectory() || entry.isSymbolicLink()) continue; + const split = splitRevision(entry.name); + if (!split) continue; + const target = path.join(holder.dir, entry.name); + members.push({ + name: entry.name, + path: target, + family: holder.scope ? `${holder.scope}/${split.family}` : split.family, + revision: split.revision, + mtimeMs: statNode(target, { fsImpl: ctx.fsImpl }).mtimeMs ?? 0, + }); + } + } + return { status: 'ok', reason: null, members }; +} + +/** + * Older revisions in each browser family — everything except the most recently + * installed one. Ordering is by directory mtime rather than by parsing the + * revision, because the revisions are not comparable across installers + * (Playwright's `1223` is a counter, its `mcp-chrome-5b42311` is a hex build id, + * Puppeteer's is a four-part Chrome version). + * + * REVIEW tier, not regenerable, even though the installer would refetch them: a + * project's pinned playwright/puppeteer version resolves to a specific revision, + * and this module cannot see which package pins what. The row's job is to say + * "these accumulated, look at them", not "delete N GB". + */ +export function browserRevisionReclaimables(ctx, roots) { + const rows = []; + for (const root of roots ?? []) { + const { status, reason, members } = revisionMembers(root, ctx); + if (status === 'absent') continue; + if (status === 'degraded') { + rows.push(unreadableCandidate({ + id: root.id, kind: root.kind, label: root.label, + target: root.path, reason, safety: 'review', + })); + continue; + } + const families = new Map(); + for (const member of members) { + const list = families.get(member.family) ?? []; + list.push(member); + families.set(member.family, list); + } + const superseded = []; + const keeps = []; + for (const [family, list] of families) { + if (list.length < 2) continue; + const ordered = [...list].sort((a, b) => b.mtimeMs - a.mtimeMs || b.name.localeCompare(a.name)); + keeps.push({ family, ...ordered[0] }); + superseded.push(...ordered.slice(1)); + } + if (!superseded.length) continue; + const { walked, capped, bytes, files } = measureMembers(superseded, ctx); + if (!worthListing(bytes)) continue; + rows.push(candidate({ + id: `${root.id}:superseded`, + kind: root.kind, + label: `${superseded.length} superseded ${root.label.toLowerCase()}`, + path: root.path, + samplePaths: walked.slice(0, ctx.opts.samplePaths).map((m) => m.path), + matchedCount: superseded.length, + bytes, + files, + safety: 'review', + keeps: keeps.map((keep) => ({ + path: keep.path, + label: `newest ${keep.family} revision (${keep.revision})`, + bytes: unknown('the retained revision is not part of this figure'), + })), + rationale: `${keeps.length} browser famil(ies) here carry more than one revision; this ` + + `figure covers the ${superseded.length} older one(s)${capped + ? `, of which ${walked.length} were measured` : ''}, and excludes the newest of each. ` + + 'Review rather than sweep: an installed package can pin an older revision, and this ' + + 'row cannot see which package pins what. ' + + `${root.installer} refetches whatever is missing.`, + cleanupHint: `${root.installer} (confirm no project pins the older revision first)`, + })); + } + return rows; +} + +// ── installed runtime versions ──────────────────────────────────────────────── + +/** + * Version-manager install roots — one directory per installed runtime version, + * plus alias entries pointing into them. + * + * @param {{ env?: NodeJS.ProcessEnv }} [options] + */ +export function runtimeVersionRoots({ env = process.env } = {}) { + return [{ + id: 'mise-installs', + kind: 'installed-runtime-versions', + label: 'mise', + path: path.join(xdgData(env), 'mise', 'installs'), + manager: 'mise', + cleanupHint: 'mise ls, then mise uninstall @ (nothing may pin it)', + }]; +} + +/** + * Installed runtime versions, one row per managed tool that has more than one. + * + * REVIEW tier and `bytesMeaning: 'installed'`, both deliberately. On this + * machine `mise/installs/node` holds eight entries — 22, 22.22, 22.22.3, 26, + * 26.4, 26.4.0, latest, lts-jod — of which only two are real directories and six + * are ALIAS SYMLINKS resolving into them. Removing a version silently breaks + * every alias that points at it, and any `mise.toml` or `.tool-versions` on the + * machine can pin any of them, so there is no subset this module can honestly + * call reclaimable. What it can do is show what is installed and say that out + * loud; recommending the deletion of a live runtime would be worse than saying + * nothing. + * + * The alias links are counted, never followed — the walker's rule, and also the + * only reason the byte figure is not multiplied by every alias. + */ +export function runtimeVersionReclaimables(ctx, roots) { + const rows = []; + for (const root of roots ?? []) { + const listing = listMembers(root.path, ctx.fsImpl); + if (listing.status === 'absent') continue; + if (listing.status === 'degraded') { + rows.push(unreadableCandidate({ + id: root.id, kind: root.kind, label: `${root.manager} installs`, + target: root.path, reason: listing.reason, safety: 'review', + })); + continue; + } + // Every managed tool is examined — listing one is a readdir, and capping the + // LIST would drop tools in alphabetical order, which is an invisible + // truncation of exactly the kind this domain forbids. What is bounded is the + // expensive part: the walks, under one budget shared across the root. + let budget = ctx.opts.maxFamilyWalks; + const tools = listing.entries + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink() + && !entry.name.startsWith('.')); + for (const tool of tools) { + const dir = path.join(root.path, tool.name); + const inner = listMembers(dir, ctx.fsImpl); + if (inner.status !== 'ok') continue; + const versions = []; + let aliases = 0; + for (const entry of inner.entries) { + if (entry.name.startsWith('.')) continue; + if (entry.isSymbolicLink()) { aliases += 1; continue; } + if (entry.isDirectory()) versions.push({ name: entry.name, path: path.join(dir, entry.name) }); + } + // One installed version is the toolchain working as intended, not sprawl. + if (versions.length < 2) continue; + const { walked, capped, bytes, files } = measureMembers(versions, ctx, budget); + budget -= walked.length; + if (!worthListing(bytes)) continue; + rows.push(candidate({ + id: `${root.id}:${tool.name}`, + kind: root.kind, + label: `${versions.length} ${root.manager} ${tool.name} versions installed`, + path: dir, + samplePaths: walked.slice(0, ctx.opts.samplePaths).map((m) => m.path), + matchedCount: versions.length, + bytes, + files, + safety: 'review', + bytesMeaning: 'installed', + rationale: `${versions.length} installed version(s) of ${tool.name}` + + `${aliases ? ` plus ${aliases} alias link(s) resolving into them` : ''}` + + `${capped ? `, of which ${walked.length} were measured` : ''}. ` + + 'This is what is installed, not what is free: an alias points at a real version, ' + + `and any ${root.manager} config on this machine can pin any of them. Review with ` + + `\`${root.manager} ls ${tool.name}\` before removing anything.`, + cleanupHint: root.cleanupHint, + })); + } + } + return rows; +} + +// ── transcripts for projects that no longer exist ───────────────────────────── + +/** + * Claude transcript directories whose project directory is gone. + * + * A `~/.claude/projects/` directory name is a LOSSY encoding of the project path + * (`/`, `.` and a literal `-` all become `-`), so it cannot be decoded by string + * manipulation; `decodeClaudeProjectDir` walks the candidate segments against the + * real filesystem and returns a path only when the filesystem confirms one. + * A directory that decodes to nothing is therefore a project that is no longer + * on this machine — and that is exactly the evidence available, so the row says + * so rather than claiming certainty. + * + * TWO GUARDS against the failure mode that would matter, flagging live projects: + * · only `-`-leading (POSIX-encoded) names are considered, because Windows + * names encode a drive letter that this decoder does not handle and would + * otherwise report every project on the machine as dead; + * · if NOTHING decoded, the finding is about the decoder or an unreadable home + * directory, not about the projects, so no row is emitted at all. + * + * The value here is hygiene and privacy, not space: measured on this machine + * these are ~0.05 GB across 8 dead projects. The rationale says that plainly — + * selling 50 MB as a disk win would be the same dishonesty as a fabricated zero, + * just in the other direction. + */ +export function orphanedTranscriptReclaimables({ + asOf, opts, transcriptProjects, decodeDir = decodeClaudeProjectDir, fsImpl, +}) { + const byHost = new Map(); + for (const entry of transcriptProjects?.values?.() ?? []) { + if (!entry.dir.startsWith('-')) continue; + const acc = byHost.get(entry.host) + ?? { host: entry.host, root: entry.rootPath, alive: 0, dead: [] }; + if (decodeDir(entry.dir, { fsImpl })) acc.alive += 1; + else acc.dead.push(entry); + byHost.set(entry.host, acc); + } + + const rows = []; + for (const acc of byHost.values()) { + if (!acc.dead.length || !acc.alive) continue; + const bytes = acc.dead.reduce((total, entry) => total + entry.bytes, 0); + const files = acc.dead.reduce((total, entry) => total + entry.files, 0); + const newest = acc.dead.reduce((at, entry) => Math.max(at, entry.newestMtimeMs ?? 0), 0); + rows.push(candidate({ + id: `orphaned-transcripts:${acc.host}`, + kind: 'orphaned-transcripts', + label: `${acc.host} transcripts for ${acc.dead.length} project(s) that no longer exist`, + path: acc.root, + samplePaths: acc.dead.slice(0, opts.samplePaths).map((entry) => entry.path), + matchedCount: acc.dead.length, + bytes: measured(bytes, { asOf }), + files: measured(files, { asOf }), + safety: 'review', + rationale: `${files} transcript(s) belong to ${acc.dead.length} project director(ies) that ` + + `no longer resolve on this machine (${acc.alive} others still do; last activity ` + + `${newest ? `${Math.floor((asOf - newest) / 86_400_000)}d ago` : 'unknown'}). This row ` + + 'is here for hygiene and privacy, not as a space win: it is listed because those ' + + 'projects are gone, whatever the byte figure turns out to be. The transcripts are also ' + + 'the only copy of that history, and an unreadable parent directory looks identical to ' + + 'a deleted project — confirm the project is really gone before acting.', + cleanupHint: null, + })); + } return rows; } @@ -584,26 +1349,25 @@ export function worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImp const record = path.join(adminRoot, entry.name); const pointer = readGitdirPointer(path.join(record, 'gitdir'), fsImpl); if (!pointer) { - rows.push({ + rows.push(candidate({ id: `orphaned-worktree:${record}`, kind: 'orphaned-worktree', label: `worktree record "${entry.name}" (unverifiable)`, path: record, - samplePaths: [], bytes: unknown('worktree pointer unreadable'), files: unknown('worktree pointer unreadable'), + safety: 'review', rationale: 'The admin record exists but its gitdir pointer could not be read, ' + 'so whether the checkout still exists is unknown.', cleanupHint: 'git worktree prune (verify first)', - advisory: true, - }); + })); continue; } // gitdir points at `/.git`; the checkout is its parent. const checkout = path.dirname(pointer); const head = statNode(checkout, { fsImpl }); if (head.status === 'unknown') { - rows.push({ + rows.push(candidate({ id: `orphaned-worktree:${record}`, kind: 'orphaned-worktree', label: `orphaned worktree record "${entry.name}"`, @@ -611,11 +1375,11 @@ export function worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImp samplePaths: [checkout], bytes: measured(0, { asOf }), files: measured(0, { asOf }), + safety: 'review', rationale: `The checkout at ${checkout} no longer exists; only the administrative ` + 'record remains.', cleanupHint: 'git worktree prune', - advisory: true, - }); + })); continue; } if (walks >= opts.maxWorktreeWalks) continue; @@ -624,7 +1388,7 @@ export function worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImp const { bytes, files } = rootMeasurements(result, { asOf }); const idleMs = result.newestMtimeMs === null ? null : asOf - result.newestMtimeMs; if (idleMs === null || idleMs < opts.worktreeIdleDays * 86_400_000) continue; - rows.push({ + rows.push(candidate({ id: `idle-worktree:${record}`, kind: 'orphaned-worktree', label: `idle worktree "${entry.name}"`, @@ -632,11 +1396,11 @@ export function worktreeReclaimables({ asOf, projects, opts, walk, limits, fsImp samplePaths: [record], bytes, files, + safety: 'review', rationale: `Nothing in the checkout has changed for ${Math.floor(idleMs / 86_400_000)}d. ` + 'Merge state is not checked here — confirm the branch is landed before removing.', cleanupHint: 'git worktree remove (verify the branch is merged first)', - advisory: true, - }); + })); } } return rows; diff --git a/src/lib/footprint/walk.mjs b/src/lib/footprint/walk.mjs index a31e121..e3fefd3 100644 --- a/src/lib/footprint/walk.mjs +++ b/src/lib/footprint/walk.mjs @@ -132,7 +132,8 @@ export function statNode(target, { fsImpl = fs } = {}) { * skipDir?: ((dir: string, name: string, depth: number) => boolean) | null, * acceptFile?: ((name: string, file: string, depth: number) => boolean) | null, * onFile?: ((entry: { file: string, name: string, bytes: number, - * mtimeMs: number, depth: number }) => void) | null, + * blocks: number, mtimeMs: number, + * depth: number }) => void) | null, * fsImpl?: typeof fs, * }} [options] `skipDir` prunes a subtree deliberately (it does NOT mark the * walk truncated — an intentional scope is not a failed measurement). @@ -183,7 +184,16 @@ export function walkTree(root, options = {}) { if (result.newestMtimeMs === null || st.mtimeMs > result.newestMtimeMs) { result.newestMtimeMs = st.mtimeMs; } - if (onFile) onFile({ file, name, bytes: st.size, mtimeMs: st.mtimeMs, depth }); + // `blocks` rides along from the stat the walk has already paid for. A + // consumer that wants to READ a file needs it: a cloud provider's evicted + // placeholder (Dropbox/iCloud/OneDrive) stats as a normal file with a real + // size but zero allocated blocks, and opening it blocks in the kernel until + // the provider materializes it — which never returns when the provider is + // signed out or offline. Undefined on a stat shim that omits it; callers + // treat only an explicit 0 as the placeholder signal. + if (onFile) { + onFile({ file, name, bytes: st.size, blocks: st.blocks, mtimeMs: st.mtimeMs, depth }); + } }; // A root that is itself a file is a legitimate node (opencode's single store, diff --git a/tests/kit/footprint-projects.test.mjs b/tests/kit/footprint-projects.test.mjs new file mode 100644 index 0000000..56fd694 --- /dev/null +++ b/tests/kit/footprint-projects.test.mjs @@ -0,0 +1,348 @@ +// Project accounting for the System area: `project-sources.mjs` (every project +// any host ever recorded a session in) and the `collectProjects` rows built on +// top of it. +// +// The contract under test is that TWO POPULATIONS stay distinct and stay +// honest. `everSeen` counts projects including the ones that have since been +// deleted — the deletions are the question, not noise — while the table only +// carries the on-disk subset, because a directory that is gone has no bytes and +// no lines to measure. A test that let those two numbers collapse into one +// would be signing off on the exact misreport this accounting exists to fix. +// +// Every fixture lives under mkdtempSync and every collector is handed explicit +// roots, so nothing here can reach the developer's real ~/.claude or ~/.codex +// and pass by accident against whatever happens to be installed. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + HEAD_MAX_LINES, PROJECT_SOURCE_HOSTS, PROJECT_SOURCE_METHOD, + decodeClaudeProjectDir, discoverProjectSources, firstCwd, scanOpencodeDirectories, + scanTranscriptCwds, +} from '../../src/lib/footprint/project-sources.mjs'; +import { collectProjects } from '../../src/lib/footprint/projects.mjs'; + +/** A fixture root that is removed when the test ends, whatever the outcome. + * Realpath'd up front: macOS /tmp is a symlink to /private/tmp, and the + * de-duplication under test resolves real paths, so a raw mkdtemp path would + * make every assertion compare two spellings of the same directory. */ +function fixture(t, name) { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-projects-${name}-`))); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +function write(file, content) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); + return file; +} + +/** One Claude transcript: flat `cwd` on its own records. */ +const claudeTranscript = (root, dirName, file, cwd) => write( + path.join(root, dirName, file), + `${JSON.stringify({ type: 'user', cwd })}\n${JSON.stringify({ type: 'assistant' })}\n`, +); + +/** One Codex rollout: `payload.cwd` on the record that opens it. */ +const codexTranscript = (root, rel, cwd) => write( + path.join(root, rel), + `${JSON.stringify({ type: 'session_meta', payload: { cwd } })}\n`, +); + +const label = (target) => path.basename(target); +const noOpencode = () => ( + { host: 'opencode', root: null, status: 'absent', reason: null, sessions: 0, sightings: [], complete: true } +); + +test('a cwd is read from either host shape, and only from the head', () => { + assert.equal(firstCwd([JSON.stringify({ cwd: '/a/b' })], 'claude'), '/a/b'); + // A Claude-shaped record read as Codex yields nothing: the shapes are not + // interchangeable and guessing across them would invent attributions. + assert.equal(firstCwd([JSON.stringify({ cwd: '/a/b' })], 'codex'), null); + assert.equal( + firstCwd([JSON.stringify({ type: 'turn_context', payload: { cwd: '/c/d' } })], 'codex'), '/c/d'); + // A truncated final line in a head window is expected, not a failure. + assert.equal(firstCwd(['{"cwd":"/x', JSON.stringify({ cwd: '/y' })], 'claude'), '/y'); + assert.equal(firstCwd([], 'claude'), null); + assert.equal(firstCwd(null, 'claude'), null); + assert.equal(firstCwd([JSON.stringify({ cwd: '' })], 'claude'), null); +}); + +test('the encoded Claude directory decodes only against the real filesystem', (t) => { + const root = fixture(t, 'decode'); + // `agentic-kit` and `agentic/kit` encode IDENTICALLY, so the decode is only + // safe because the filesystem decides which one exists. + fs.mkdirSync(path.join(root, 'ai', 'agentic-kit'), { recursive: true }); + const encoded = path.join(root, 'ai', 'agentic-kit').split(path.sep).join('-'); + assert.equal(decodeClaudeProjectDir(encoded), path.join(root, 'ai', 'agentic-kit')); + + // A dotted directory is the other separator the encoding swallows. + fs.mkdirSync(path.join(root, '.claude', 'x'), { recursive: true }); + const dotted = path.join(root, '.claude', 'x').split(path.sep).join('-'); + assert.equal(decodeClaudeProjectDir(dotted), path.join(root, '.claude', 'x')); + + // A project whose directory is GONE cannot be recovered — stated as null, not + // guessed at, which is what makes `everSeen` a floor rather than a fiction. + assert.equal(decodeClaudeProjectDir(`${root.split(path.sep).join('-')}-vanished`), null); + assert.equal(decodeClaudeProjectDir('not-absolute'), null); + assert.equal(decodeClaudeProjectDir(''), null); +}); + +test('one project touched by two hosts is ONE project, resolved through symlinks', (t) => { + const root = fixture(t, 'dedup'); + const real = path.join(root, 'work', 'shared-repo'); + fs.mkdirSync(path.join(real, '.git'), { recursive: true }); + const link = path.join(root, 'link-to-shared'); + fs.symlinkSync(real, link, 'dir'); + + const claudeRoot = path.join(root, 'claude-projects'); + const codexRoot = path.join(root, 'codex-sessions'); + // Same project, three spellings: the real path, a symlinked path, and a + // second Claude session in the same directory. + claudeTranscript(claudeRoot, '-work-shared-repo', 'a.jsonl', real); + claudeTranscript(claudeRoot, '-work-shared-repo', 'b.jsonl', link); + codexTranscript(codexRoot, path.join('2026', '08', '06', 'r.jsonl'), real); + + const found = discoverProjectSources({ + claudeRoot, codexRoot, resolveLabel: label, + scanOpencode: noOpencode, now: () => 1_700_000_000_000, + }); + + assert.equal(found.everSeen, 1, JSON.stringify(found.projects, null, 2)); + assert.equal(found.onDisk, 1); + assert.equal(found.gitRepos, 1); + assert.deepEqual(found.projects[0].hosts, ['claude', 'codex']); + assert.equal(found.projects[0].path, real); + assert.equal(found.projects[0].sessions, 3); + assert.equal(found.projects[0].exists, true); + assert.equal(found.method, PROJECT_SOURCE_METHOD); + assert.deepEqual(Object.keys(found.sources).sort(), [...PROJECT_SOURCE_HOSTS].sort()); +}); + +test('a project that no longer exists is counted in everSeen and never in the table', (t) => { + const root = fixture(t, 'vanished'); + const alive = path.join(root, 'alive'); + fs.mkdirSync(path.join(alive, '.git'), { recursive: true }); + const gone = path.join(root, 'deleted-last-week'); + + const claudeRoot = path.join(root, 'claude-projects'); + claudeTranscript(claudeRoot, '-alive', 'a.jsonl', alive); + claudeTranscript(claudeRoot, '-deleted-last-week', 'b.jsonl', gone); + + const sources = discoverProjectSources({ + claudeRoot, + codexRoot: path.join(root, 'no-codex'), + resolveLabel: label, + scanOpencode: noOpencode, + now: () => 1_700_000_000_000, + }); + assert.equal(sources.everSeen, 2); + assert.equal(sources.onDisk, 1); + assert.equal(sources.gitRepos, 1); + // The vanished project keeps its row in DISCOVERY — that is where the fact + // that it existed lives — and is honestly marked as not on disk. + const vanished = sources.projects.find((p) => p.path === gone); + assert.equal(vanished.exists, false); + assert.equal(vanished.isGitRepo, false); + // An absent codex root is an ABSENCE (that host was never used here), not a + // failed measurement, so it must not make the counts a floor. + assert.equal(sources.sources.codex.status, 'absent'); + assert.equal(sources.complete, true); + + const section = collectProjects({ + sources, loc: false, now: () => 1_700_000_000_000, + }); + assert.equal(section.projects.length, 1, 'only the measurable project becomes a row'); + assert.equal(section.projects[0].path, alive); + assert.equal(section.everSeen.value, 2); + assert.equal(section.onDisk.value, 1); + assert.equal(section.gitRepos.value, 1); + // `count` keeps its original meaning for existing consumers: how many + // projects discovery found. + assert.equal(section.count.value, section.everSeen.value); + assert.equal(section.method, PROJECT_SOURCE_METHOD); + assert.equal(section.everSeen.partial, false); + assert.deepEqual(section.projects[0].hosts, ['claude']); +}); + +test('an unreadable transcript makes the counts a floor, never a smaller total', (t) => { + const root = fixture(t, 'floor'); + const claudeRoot = path.join(root, 'claude-projects'); + claudeTranscript(claudeRoot, '-p', 'a.jsonl', path.join(root, 'p')); + + // `null` from the head reader is "could not read at all", which is distinct + // from an empty file — an empty file is a real, readable zero lines. + const unreadable = scanTranscriptCwds(claudeRoot, 'claude', { readHead: () => null }); + assert.equal(unreadable.files, 1); + assert.equal(unreadable.unreadable, 1); + assert.equal(unreadable.withCwd, 0); + assert.equal(unreadable.complete, false); + + const empty = scanTranscriptCwds(claudeRoot, 'claude', { readHead: () => [] }); + assert.equal(empty.empty, 1); + assert.equal(empty.unreadable, 0); + assert.equal(empty.complete, true); + + // A directory group with no cwd anywhere and no recoverable path is the other + // way the list becomes a floor. + const lost = scanTranscriptCwds(claudeRoot, 'claude', { + readHead: () => ['{}'], decodeDir: () => null, + }); + assert.equal(lost.withoutCwd, 1); + assert.equal(lost.unresolved, 1); + assert.equal(lost.complete, false); + + const section = collectProjects({ + discover: () => ({ + projects: [], everSeen: 4, onDisk: 0, gitRepos: 0, unresolved: 1, + complete: false, method: PROJECT_SOURCE_METHOD, sources: {}, + }), + loc: false, + now: () => 1_700_000_000_000, + }); + assert.equal(section.everSeen.value, 4); + assert.equal(section.everSeen.partial, true, 'an incomplete sweep renders as ">= N"'); + assert.equal(section.unresolved, 1); + assert.equal(section.complete, false); +}); + +test('a group with no cwd anywhere is recovered from its encoded directory name', (t) => { + const root = fixture(t, 'recover'); + const project = path.join(root, 'quiet-project'); + fs.mkdirSync(project, { recursive: true }); + const claudeRoot = path.join(root, 'claude-projects'); + const encoded = project.split(path.sep).join('-'); + // No cwd record anywhere in this group — the directory name is the only + // evidence left, and it is only usable because the directory still exists. + write(path.join(claudeRoot, encoded, 'a.jsonl'), `${JSON.stringify({ type: 'user' })}\n`); + + const scan = scanTranscriptCwds(claudeRoot, 'claude', { + decodeDir: (name) => decodeClaudeProjectDir(name), + }); + assert.equal(scan.withCwd, 0); + assert.equal(scan.recoveredFromDirName, 1); + assert.equal(scan.unresolved, 0); + assert.deepEqual(scan.sightings.map((s) => s.origin), ['encoded-dir']); + assert.equal(scan.sightings[0].cwd, project); + assert.equal(scan.complete, true); +}); + +test('OpenCode sessions come from the store, and a broken store degrades with its reason', () => { + const rows = [ + { directory: '/w/one', sessions: 3, lastMs: 1_700_000_000_000 }, + { directory: '/w/two', sessions: 1, lastMs: null }, + ]; + const ok = scanOpencodeDirectories({ + dbFile: '/nowhere/opencode.db', + withDb: () => ({ ok: true, value: rows }), + }); + assert.equal(ok.status, 'ok'); + assert.equal(ok.sessions, 4); + assert.deepEqual(ok.sightings.map((s) => s.weight), [3, 1]); + assert.equal(ok.complete, true); + + const absent = scanOpencodeDirectories({ + dbFile: '/nowhere/opencode.db', + withDb: () => ({ ok: false, error: { kind: 'absent' } }), + }); + assert.equal(absent.status, 'absent'); + assert.equal(absent.complete, true, 'never used here is a real zero, not a failure'); + + const broken = scanOpencodeDirectories({ + dbFile: '/nowhere/opencode.db', + withDb: () => ({ ok: false, error: { kind: 'io', message: 'SQLITE_CORRUPT' } }), + }); + assert.equal(broken.status, 'degraded'); + assert.equal(broken.reason, 'SQLITE_CORRUPT'); + assert.equal(broken.complete, false); +}); + +test('an OpenCode-only project joins the same de-duplicated list', (t) => { + const root = fixture(t, 'opencode'); + const shared = path.join(root, 'shared'); + const oc = path.join(root, 'opencode-only'); + fs.mkdirSync(shared, { recursive: true }); + fs.mkdirSync(oc, { recursive: true }); + const claudeRoot = path.join(root, 'claude-projects'); + claudeTranscript(claudeRoot, '-shared', 'a.jsonl', shared); + + const found = discoverProjectSources({ + claudeRoot, + codexRoot: path.join(root, 'no-codex'), + resolveLabel: label, + scanOpencode: () => ({ + host: 'opencode', root: 'db', status: 'ok', reason: null, sessions: 5, complete: true, + sightings: [ + { cwd: shared, mtimeMs: 2, origin: 'cwd', weight: 2 }, + { cwd: oc, mtimeMs: 1, origin: 'cwd', weight: 3 }, + ], + }), + now: () => 1_700_000_000_000, + }); + assert.equal(found.everSeen, 2); + const sharedRow = found.projects.find((p) => p.path === shared); + assert.deepEqual(sharedRow.hosts, ['claude', 'opencode']); + assert.equal(sharedRow.sessions, 3, 'the OpenCode row carries its own session weight'); + assert.deepEqual(found.projects.find((p) => p.path === oc).hosts, ['opencode']); +}); + +test('an explicit catalog is measured exactly as given, existence and all', (t) => { + const root = fixture(t, 'explicit'); + const alive = path.join(root, 'alive'); + fs.mkdirSync(path.join(alive, '.git'), { recursive: true }); + + const section = collectProjects({ + projects: [ + { path: alive, label: 'alive' }, + { path: path.join(root, 'gone'), label: 'gone' }, + ], + loc: false, + now: () => 1_700_000_000_000, + }); + // The caller — not discovery — chose these rows, so the on-disk filter does + // NOT apply and the missing one is reported as a missing row rather than + // silently dropped. + assert.equal(section.projects.length, 2); + assert.equal(section.everSeen.value, 2); + assert.equal(section.onDisk.value, 1); + assert.equal(section.gitRepos.value, 1); + assert.equal(section.method, null, 'an explicit catalog has no discovery method to state'); +}); + +test('discovery that throws reports unknown counts, never zeros', () => { + const section = collectProjects({ + discover: () => { const error = new Error('nope'); error.code = 'EACCES'; throw error; }, + loc: false, + now: () => 1_700_000_000_000, + }); + assert.equal(section.projects.length, 0); + for (const key of ['count', 'everSeen', 'onDisk', 'gitRepos']) { + assert.equal(section[key].value, null, `${key} must not fabricate a 0`); + assert.equal(section[key].status, 'unknown'); + assert.equal(section[key].reason, 'EACCES'); + } + assert.equal(section.complete, false); +}); + +test('only the head of a transcript is read, and the budget is stated', (t) => { + const root = fixture(t, 'head'); + const claudeRoot = path.join(root, 'claude-projects'); + // The cwd sits far past the line budget: a session records its cwd in its + // opening records or not at all, so reading further would cost the whole + // corpus to learn nothing. + const filler = `${JSON.stringify({ type: 'assistant' })}\n`.repeat(HEAD_MAX_LINES + 10); + write(path.join(claudeRoot, '-late', 'a.jsonl'), + `${filler}${JSON.stringify({ cwd: path.join(root, 'late') })}\n`); + + const scan = scanTranscriptCwds(claudeRoot, 'claude', { decodeDir: () => null }); + assert.equal(scan.withCwd, 0); + assert.equal(scan.withoutCwd, 1); + + // The same file is found when the budget is widened — proving the miss above + // is the bound doing its job rather than a parser bug. + const wide = scanTranscriptCwds(claudeRoot, 'claude', { maxLines: HEAD_MAX_LINES + 20 }); + assert.equal(wide.withCwd, 1); +}); diff --git a/tests/kit/footprint-stack.test.mjs b/tests/kit/footprint-stack.test.mjs new file mode 100644 index 0000000..9a845a0 --- /dev/null +++ b/tests/kit/footprint-stack.test.mjs @@ -0,0 +1,388 @@ +// The stack registry and the detection built on it: which languages hold a +// project's lines, which frameworks/SDKs/tools it declares, and everything the +// registry could not name. +// +// Three properties carry the whole design, and each is asserted rather than +// assumed: +// +// LINES BELONG TO LANGUAGES ONLY. A framework/sdk/tool entry must never carry +// a line count anywhere — in the registry or in a detection payload. react +// does not own lines, the .tsx files do, and a bar that stacked both would +// count the same bytes twice. +// +// THE UNRECOGNIZED TAIL IS REAL AND RANKED. An extension the registry does +// not know is never counted as lines and never silently dropped: it is tallied +// by name so "Other" is a to-do list a release can close. +// +// MANIFESTS ARE READ SHALLOWLY. A vendored or deeply nested dependency's own +// manifest must not be mistaken for the project's declarations — on a real +// machine nearly every Cargo.toml belongs to a registry cache, not a project. +// +// Every fixture lives under mkdtempSync; the registry half needs no filesystem +// at all, which is the point of it being pure data. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + COLOR_SLOTS, ECOSYSTEMS, MANIFEST_KINDS, NON_SOURCE_EXTENSIONS, REGISTRY_CONFLICTS, + STACK_REGISTRY, STACK_REGISTRY_VERSION, dependencyEntry, isNonSourceExtension, + languageForExtension, languageForFilename, manifestKindFor, registryStats, + signatureEntries, stackEntries, stackEntryById, +} from '../../src/lib/footprint/stack-registry.mjs'; +import { + EXCLUDED_DIRS, EXCLUDED_FILES, MANIFEST_MAX_DEPTH, STACK_EXCLUSIONS, + detectStack, parseManifestDependencies, stackProvenance, +} from '../../src/lib/footprint/stack-detect.mjs'; + +function fixture(t, name) { + const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-stack-${name}-`))); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +function write(root, rel, content) { + const file = path.join(root, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content); + return file; +} + +const ids = (rows) => rows.map((row) => row.id); + +// ── the registry as pure data ───────────────────────────────────────────────── + +test('the registry indexes cleanly — no shadowed entry, no double-decided extension', () => { + assert.deepEqual(REGISTRY_CONFLICTS, [], + `duplicate registry keys: ${JSON.stringify(REGISTRY_CONFLICTS)}`); + assert.equal(new Set(STACK_REGISTRY.map((e) => e.id)).size, STACK_REGISTRY.length); + assert.match(STACK_REGISTRY_VERSION, /^\d{4}\.\d{2}\.\d+$/); +}); + +test('every entry stays inside the closed vocabularies it declares against', () => { + for (const entry of STACK_REGISTRY) { + assert.ok(ECOSYSTEMS.includes(entry.ecosystem), `${entry.id}: ecosystem ${entry.ecosystem}`); + assert.ok(['language', 'framework', 'sdk', 'tool'].includes(entry.kind), `${entry.id}`); + assert.ok(Object.isFrozen(entry), `${entry.id} must be frozen data`); + if (entry.match.by === 'dependency') { + for (const manifest of entry.match.manifests) { + assert.ok(MANIFEST_KINDS.includes(manifest), `${entry.id}: manifest ${manifest}`); + } + } + } +}); + +test('a framework, sdk or tool NEVER owns lines — only languages do', () => { + for (const entry of STACK_REGISTRY) { + if (entry.kind === 'language') { + assert.equal(entry.match.by, 'extension', `${entry.id} is a language but not extension-matched`); + assert.ok(COLOR_SLOTS.includes(entry.colorSlot), `${entry.id}: slot ${entry.colorSlot}`); + assert.ok(entry.match.extensions.length || entry.match.filenames.length, entry.id); + continue; + } + // The structural guarantee: a non-language entry has no extensions and no + // colour slot, so there is nothing for a renderer to put on a lines bar. + assert.notEqual(entry.match.by, 'extension', `${entry.id} must not claim extensions`); + assert.equal(entry.match.extensions, undefined, `${entry.id} must not carry extensions`); + assert.equal(entry.colorSlot, undefined, `${entry.id} must not carry a lines colour slot`); + assert.equal(entry.lines, undefined, `${entry.id} must not carry lines`); + } + // Vue is deliberately TWO entries — the language that owns `.vue` lines and + // the framework whose presence is a separate fact. + assert.equal(stackEntryById('vue').kind, 'language'); + assert.equal(stackEntryById('vue-framework').kind, 'framework'); + assert.equal(stackEntryById('nope'), null); +}); + +test('extension, filename and manifest lookups answer null rather than guessing', () => { + assert.equal(languageForExtension('.TSX').id, 'typescript'); + assert.equal(languageForExtension('.rs').id, 'rust'); + assert.equal(languageForExtension('.zzz'), null, 'unknown is null, not a zero bucket'); + assert.equal(languageForExtension(null), null); + assert.equal(languageForFilename('makefile').id, 'makefile'); + assert.equal(languageForFilename('Dockerfile').id, 'dockerfile'); + assert.equal(languageForFilename('README'), null); + + assert.equal(manifestKindFor('package.json'), 'npm'); + assert.equal(manifestKindFor('Cargo.toml'), 'cargo'); + assert.equal(manifestKindFor('requirements-dev.txt'), 'python'); + assert.equal(manifestKindFor('Gemfile'), 'rubygems'); + assert.equal(manifestKindFor('notes.txt'), null); + + // A name only means something inside ONE manifest family: `openai` is a + // different package on npm and on PyPI, and `react` is not a Python package. + assert.equal(dependencyEntry('npm', 'react').id, 'react'); + assert.equal(dependencyEntry('python', 'react'), null); + assert.equal(dependencyEntry('python', 'FastAPI').id, 'fastapi'); + // Longest prefix wins, so a scoped family never loses to a shorter neighbour. + assert.equal(dependencyEntry('npm', '@langchain/core').id, 'langchain'); + assert.equal(dependencyEntry('npm', 'not-a-real-package'), null); + assert.equal(dependencyEntry('npm', ''), null); + + // A stated exclusion is a decision; an unmapped extension is a gap. The two + // must not be the same answer. + assert.equal(isNonSourceExtension('.PNG'), true); + assert.equal(isNonSourceExtension('.zzz'), false); + assert.equal(new Set(NON_SOURCE_EXTENSIONS).size, NON_SOURCE_EXTENSIONS.length); +}); + +test('registryStats is the provenance a panel prints, and counts what is there', () => { + const stats = registryStats(); + assert.equal(stats.version, STACK_REGISTRY_VERSION); + assert.equal(stats.entries, STACK_REGISTRY.length); + assert.equal(stats.languages, stackEntries('language').length); + assert.equal(stats.languages + stats.frameworks + stats.sdks + stats.tools, stats.entries); + assert.ok(stats.extensions > 100, `only ${stats.extensions} extensions indexed`); + assert.equal(stats.nonSourceExtensions, NON_SOURCE_EXTENSIONS.length); + assert.ok(signatureEntries().every((entry) => entry.match.by === 'signature')); +}); + +// ── manifest parsing, without a filesystem ─────────────────────────────────── + +test('manifest parsers read KEYS only, and never evaluate their input', () => { + assert.deepEqual( + parseManifestDependencies('npm', JSON.stringify({ + dependencies: { react: '^18', local: 'workspace:*' }, + devDependencies: { vitest: '^1' }, + })), + ['react', 'vitest'], 'a workspace protocol is the project depending on itself'); + assert.equal(parseManifestDependencies('npm', 'not json'), null, + 'unparseable is null — reported as degraded, never as "declares nothing"'); + + // Cargo: a path dependency is a workspace member, not a third-party crate. + assert.deepEqual( + parseManifestDependencies('cargo', + '[dependencies]\ntokio = "1"\nsibling = { path = "crates/x" }\n[dev-dependencies]\ncriterion = "0"\n'), + ['tokio', 'criterion']); + + // pyproject vs requirements.txt share a manifest kind and nothing else; the + // leading `[` is the discriminator, because no requirement line can start + // with one. + assert.deepEqual( + parseManifestDependencies('python', '[project]\ndependencies = ["FastAPI>=0.1", "httpx"]\n'), + ['fastapi', 'httpx']); + assert.deepEqual( + parseManifestDependencies('python', '# comment\nDjango==5.0\n-r other.txt\nboto3\n'), + ['django', 'boto3']); + + // go.mod: an indirect requirement is a transitive dependency, not this + // project's stack. + assert.deepEqual( + parseManifestDependencies('gomod', + 'require github.com/spf13/cobra v1.8.0\nrequire golang.org/x/net v0.1.0 // indirect\n'), + ['github.com/spf13/cobra']); + + assert.ok(parseManifestDependencies('jvm', + 'org.springframework.bootspring-boot-starter') + .includes('org.springframework.boot')); + assert.deepEqual(parseManifestDependencies('hex', '[{:phoenix, "~> 1.7"}, {:ecto, "~> 3.0"}]'), + ['phoenix', 'ecto']); + assert.deepEqual(parseManifestDependencies('rubygems', "gem 'rails', '~> 7'\n# gem 'nope'\n"), + ['rails']); + assert.deepEqual( + parseManifestDependencies('pub', 'dependencies:\n flutter:\n sdk: flutter\n dio: ^5\n'), + ['flutter', 'dio']); + assert.equal(parseManifestDependencies('unheard-of', 'anything'), null); +}); + +// ── detection over a real project ──────────────────────────────────────────── + +/** A project with lines in two languages, one recognized framework, one + * unrecognized dependency, an unrecognized-extension tail, a stated non-source + * file, and manifests at every depth the bound cares about. */ +function project(t) { + const root = fixture(t, 'detect'); + write(root, 'package.json', JSON.stringify({ + dependencies: { react: '^18', 'totally-unknown-lib': '^1' }, + })); + write(root, 'src/app.tsx', 'a\nb\nc\n'); + write(root, 'src/util.ts', 'x\ny\n'); + write(root, 'src/index.js', 'one\n'); + // Unrecognized tail: three of one extension, one of another, so the ranking + // has something to rank. + write(root, 'a.zeta', '1\n'); + write(root, 'b.zeta', '1\n'); + write(root, 'c.zeta', '1\n'); + write(root, 'd.qux', '1\n'); + // A stated non-source exclusion must NOT reach the tail. + write(root, 'logo.png', Buffer.from([0x89, 0x50, 0x4e, 0x47])); + // A rotated log's "extension" is not shaped like one; it collapses into a + // named bucket rather than minting a single-file row. + write(root, 'debug.2026-08-06', 'x\n'); + write(root, 'LICENSE', 'x\n'); + // A lockfile is a deliberate exclusion, not a to-do. + write(root, 'pnpm-lock.yaml', 'lockfileVersion: 9\n'); + // Signature: the deepest one the scan looks for. + write(root, '.github/workflows/ci.yml', 'name: ci\n'); + // A monorepo package manifest is the project's own and IS read. + write(root, 'packages/api/package.json', JSON.stringify({ dependencies: { fastify: '^4' } })); + // A vendored dependency's manifest is NOT: `vendor/` is pruned outright... + write(root, 'vendor/thing/package.json', JSON.stringify({ dependencies: { express: '^4' } })); + write(root, 'node_modules/dep/package.json', JSON.stringify({ dependencies: { koa: '^2' } })); + // ...and a manifest deeper than the bound is out of scope even in a tree + // nobody prunes. + write(root, 'deep/a/b/package.json', JSON.stringify({ dependencies: { angular: '^17' } })); + return root; +} + +test('lines land in languages, and the stack payload has no lines field at all', (t) => { + const root = project(t); + const out = detectStack(root, { asOf: 1_700_000_000_000 }); + + assert.equal(out.registryVersion, STACK_REGISTRY_VERSION); + assert.equal(out.approximate, true); + const byId = new Map(out.languages.map((row) => [row.id, row])); + assert.equal(byId.get('typescript').lines, 5, 'app.tsx + util.ts'); + assert.equal(byId.get('typescript').files, 2); + assert.equal(byId.get('javascript').lines, 1); + // Ranked by lines, so the biggest bucket reads first. `config` is in this + // list too: a manifest is read for its dependency keys AND holds lines, and + // the registry maps `.json` to a language for exactly that reason. + assert.equal(out.languages[0].id, 'typescript'); + assert.ok(out.languages.some((row) => row.id === 'javascript')); + for (let i = 1; i < out.languages.length; i++) { + assert.ok(out.languages[i - 1].lines >= out.languages[i].lines, 'ranked by lines'); + } + assert.equal(out.totalLines.status, 'measured'); + assert.equal(out.totalLines.value, + out.languages.reduce((sum, row) => sum + row.lines, 0)); + assert.equal(out.totalLines.partial, false); + + // The load-bearing assertion: presence entries are structurally incapable of + // carrying a line count. + assert.ok(out.stack.length > 0); + for (const item of out.stack) { + assert.ok(!('lines' in item), `${item.id} carries a lines field`); + assert.ok(!('files' in item), `${item.id} carries a files field`); + assert.notEqual(item.kind, 'language', `${item.id} is a language on the presence list`); + assert.deepEqual(Object.keys(item).sort(), ['ecosystem', 'id', 'kind', 'name', 'via']); + } + assert.ok(ids(out.stack).includes('react')); + assert.ok(ids(out.stack).includes('github-actions'), 'a directory signature is presence too'); + assert.ok(!ids(out.stack).includes('typescript'), 'the language bucket is not a stack row'); +}); + +test('a vendored or too-deep manifest is never mistaken for the project\'s own', (t) => { + const root = project(t); + const out = detectStack(root, { asOf: 1 }); + const seen = ids(out.stack); + + assert.ok(seen.includes('react'), 'the root manifest is the project\'s own'); + assert.ok(seen.includes('fastify'), 'a monorepo package manifest is in scope'); + assert.ok(!seen.includes('express'), 'vendor/ is pruned, so its manifest never appears'); + assert.ok(!seen.includes('koa'), 'node_modules is pruned'); + assert.ok(!seen.includes('angular'), `a manifest below depth ${MANIFEST_MAX_DEPTH} is out of scope`); + + const manifestPaths = out.manifests.map((row) => path.relative(root, row.path)); + assert.deepEqual(manifestPaths.sort(), ['package.json', path.join('packages', 'api', 'package.json')]); + assert.ok(out.manifests.every((row) => row.status === 'read')); + + // Turning manifests off leaves signature-detected rows and nothing else, so + // the two detection paths are provably independent. + const noManifests = detectStack(root, { manifests: false, asOf: 1 }); + assert.deepEqual(noManifests.manifests, []); + assert.ok(!ids(noManifests.stack).includes('react')); + assert.ok(ids(noManifests.stack).includes('github-actions')); +}); + +test('the unrecognized tail is populated, ranked, and admits only real gaps', (t) => { + const root = project(t); + const out = detectStack(root, { asOf: 1 }); + const tail = out.unrecognized.extensions; + const tailByExt = new Map(tail.map((row) => [row.ext, row])); + + assert.equal(tailByExt.get('.zeta').files, 3); + assert.equal(tailByExt.get('.qux').files, 1); + // Ranked by file count, so the biggest gap is the first thing a release sees. + assert.equal(tail[0].ext, '.zeta'); + assert.ok(tail.findIndex((row) => row.ext === '.zeta') + < tail.findIndex((row) => row.ext === '.qux')); + assert.equal(out.unrecognized.extensionsTotal, tail.length); + + // A stated exclusion is counted separately and never pollutes the to-do list. + assert.ok(!tailByExt.has('.png')); + assert.equal(out.nonSource.files, 1); + assert.ok(out.nonSource.bytes > 0); + // A rotated log collapses into a named bucket rather than minting a row. + assert.ok(!tailByExt.has('.2026-08-06')); + assert.ok(tailByExt.has('(other)')); + assert.equal(tailByExt.get('(no extension)').files, 1, 'LICENSE'); + // Lockfiles are excluded outright — not lines, and not a gap either. + assert.ok(!tailByExt.has('.yaml'), 'pnpm-lock.yaml is a stated exclusion'); + + // Unrecognized DEPENDENCIES are the other half of the same to-do list. + const unknownDeps = out.unrecognized.dependencies.map((row) => row.name); + assert.ok(unknownDeps.includes('totally-unknown-lib')); + assert.ok(!unknownDeps.includes('react')); + assert.equal(out.unrecognized.dependenciesTotal, out.unrecognized.dependencies.length); + assert.ok(out.unrecognized.dependencies.every((row) => MANIFEST_KINDS.includes(row.manifest))); +}); + +test('an unwalkable project is unknown with a reason, never a measured zero', () => { + const out = detectStack(path.join(os.tmpdir(), 'ak-fp-stack-does-not-exist-9x8y7z'), { asOf: 5 }); + assert.equal(out.totalLines.status, 'unknown'); + assert.equal(out.totalLines.value, null); + assert.equal(out.totalLines.reason, 'ENOENT'); + assert.deepEqual(out.languages, []); + assert.deepEqual(out.stack, []); + assert.equal(out.files, null, 'a file count nobody took is null, not 0'); + assert.equal(out.complete, false); + assert.ok(out.exclusions.length, 'the scope is stated even when nothing was measured'); +}); + +test('an empty project is a measured zero, which is a different answer', (t) => { + const root = fixture(t, 'empty'); + const out = detectStack(root, { asOf: 7 }); + assert.equal(out.totalLines.status, 'measured'); + assert.equal(out.totalLines.value, 0); + assert.equal(out.files, 0); + assert.equal(out.complete, true); +}); + +test('a binary or oversized file is skipped and counted, not line-counted', (t) => { + const root = fixture(t, 'skips'); + // A `.js` file with a NUL byte is not source anyone wrote lines in. + fs.writeFileSync(path.join(root, 'blob.js'), Buffer.from([0x61, 0x00, 0x62, 0x0a])); + write(root, 'real.js', 'a\nb\n'); + const out = detectStack(root, { asOf: 1 }); + assert.equal(out.skipped, 1); + assert.equal(out.languages.find((row) => row.id === 'javascript').lines, 2); + assert.equal(out.files, 1, 'only the counted file is a counted file'); + // A file with no trailing newline still ends a line. + write(root, 'tail.js', 'a\nb'); + assert.equal(detectStack(root, { asOf: 1 }).languages + .find((row) => row.id === 'javascript').lines, 4); +}); + +test('a capped walk makes the total a floor rather than a smaller number', (t) => { + const root = project(t); + const capped = detectStack(root, { limits: { maxEntries: 3 }, asOf: 9 }); + assert.equal(capped.totalLines.status, 'measured'); + assert.equal(capped.totalLines.partial, true, 'a capped total renders as ">= N"'); + assert.equal(capped.complete, false); +}); + +test('the exclusions and the provenance travel with the figure', (t) => { + const root = project(t); + const out = detectStack(root, { asOf: 1 }); + for (const dir of EXCLUDED_DIRS) assert.ok(STACK_EXCLUSIONS.includes(`${dir}/`), dir); + for (const file of EXCLUDED_FILES) assert.ok(STACK_EXCLUSIONS.includes(file), file); + assert.deepEqual(out.exclusions, [...STACK_EXCLUSIONS]); + + const note = stackProvenance(out); + assert.equal(note.registryVersion, STACK_REGISTRY_VERSION); + assert.equal(note.languages, out.languages.length); + assert.equal(note.stackItems, out.stack.length); + assert.equal(note.manifestsRead, 2); + assert.equal(note.manifestsSeen, out.manifests.length); + assert.equal(note.nonSourceFiles, 1); + assert.equal(note.approximate, true); + + // Provenance for a project that was never measured says so rather than + // reporting a confident zero-language stack. + const none = stackProvenance(null); + assert.equal(none.languages, 0); + assert.equal(none.unrecognizedExtensions, null); + assert.equal(none.nonSourceFiles, null); +}); From fdd70520a9ee5868e5e8b0c40f537dbebe3fca2f Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Sat, 8 Aug 2026 08:49:45 -0700 Subject: [PATCH 14/19] feat(dashboard): one project census, current models, reworked System area MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses a review of the local diagnostic panel. Four themes. Project counting. Overview/Usage/Observability/System each discovered projects their own way and reported four different numbers for the same machine (4, 14, another 14, ~48). ADR-0027 makes discoverProjectSources() the single census with four named scopes, and no surface may render a project count without the sentence explaining what it counted. The Intelligence panel now asks whether memory/intelligence has been ACTIVATED (.claude-flow, .agentic-qe or .swarm, any host) rather than whether ruflo has trained: 4 projects becomes 17. The learning scope folds a repo's sub-directories and agent worktrees onto one identity — without that, keying the picker off identity left 7 of 24 rows unreachable. Retires project-discovery.mjs; registryWorkspaces() is module-private again. Models. gpt-5.4/gpt-5.4-mini retire from Codex on 2026-08-31 and gpt-5.3-codex is already withdrawn, so execution routes to gpt-5.6-terra and mechanical work to gpt-5.6-luna. RETIRED_MODELS substitutes a withdrawn model at the read boundary — the one place a user pin is overridden, because honoring a pin into a dead model fails the run — and ak sync rewrites seeded routes naming one. claude-opus-4-8 is deliberately NOT listed: it carries no deprecation notice, so it is divergence, not retirement. System area. Seven sub-views: Advisory and Sessions split out of Storage, Advisory because it is the only part of System that suggests an action. Storage lifts learning stores (99% of retained bytes) onto their own card so the donut is legible, restricts the per-host split to real hosts, and gives growth five axed sparklines. Catalog covers project scope across every project on disk and gains kind/host filters. Projects lists only repositories with a remote that a host has recorded a session in. Honest degradation. Removes the AI-worker budget tile: no code path could ever populate it, so it was a permanent "unavailable" rather than a degradation. ADR-0023 gains §9 (a permanently unmeasurable quantity is deleted, not degraded) and §10 (an excluded figure is still stated). Also fixes eslint linting gitignored .ui-artifacts/, which failed `pnpm run check` for anyone who had run `pnpm test:ui`. --- docs/DASHBOARD.md | 44 +- docs/PROVIDERS.md | 29 +- docs/TRANSCRIPTS.md | 60 +- docs/UPGRADING.md | 15 + docs/USAGE-SCORECARD-METRICS.md | 110 +-- ...-vocabulary-defaults-from-ruv-templates.md | 7 + .../0003-auto-seed-dual-host-provenance.md | 43 +- ...sed-operations-and-explicit-degradation.md | 26 + .../0024-project-intelligence-telemetry.md | 23 +- docs/adr/0025-machine-footprint-metrics.md | 22 +- docs/adr/0027-shared-project-census.md | 105 +++ docs/adr/README.md | 17 + docs/ddd/context-map.md | 21 +- docs/ddd/machine-footprint.md | 59 +- docs/ddd/project-intelligence.md | 79 +- docs/ddd/routing-and-orchestration.md | 18 + docs/ddd/ubiquitous-language.md | 18 + eslint.config.mjs | 5 + src/commands/sync.mjs | 16 +- src/lib/daemons.mjs | 14 +- src/lib/dashboard-server.mjs | 66 +- src/lib/dashboard/client.mjs | 867 +++++++++++++----- src/lib/dashboard/intel-history.mjs | 52 +- src/lib/dashboard/page.mjs | 159 +++- src/lib/dashboard/project-discovery.mjs | 254 ----- src/lib/dashboard/styles.mjs | 155 +++- src/lib/footprint/catalog.mjs | 33 +- src/lib/footprint/index.mjs | 10 +- src/lib/footprint/runtime.mjs | 28 +- src/lib/footprint/storage.mjs | 46 +- src/lib/pricing.mjs | 6 + src/lib/project-census.mjs | 197 ++++ src/lib/providers.mjs | 16 +- src/lib/routing.mjs | 165 +++- src/lib/usage-index.mjs | 63 +- .../dashboard-integration-identity.test.mjs | 40 +- tests/kit/footprint-collectors.test.mjs | 67 +- tests/kit/footprint-windows.test.mjs | 7 +- tests/kit/ga-surface-guard.test.mjs | 16 + tests/kit/intel-history.test.mjs | 27 +- tests/kit/project-census.test.mjs | 258 ++++++ tests/kit/project-discovery.test.mjs | 304 ------ tests/kit/routing-divergence.test.mjs | 25 +- tests/kit/routing-retirement.test.mjs | 210 +++++ tests/ui/dashboard-ui.mjs | 221 ++++- 45 files changed, 2872 insertions(+), 1151 deletions(-) create mode 100644 docs/adr/0027-shared-project-census.md delete mode 100644 src/lib/dashboard/project-discovery.mjs create mode 100644 src/lib/project-census.mjs create mode 100644 tests/kit/project-census.test.mjs delete mode 100644 tests/kit/project-discovery.test.mjs create mode 100644 tests/kit/routing-retirement.test.mjs diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index d659721..c9bff3b 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -36,7 +36,7 @@ permanent. | Overview | Hosts & Routing | `#overview/hosts` | Hosts & routing | Enabled execution hosts, activity assignments, primary-host policy, and escalation paths | | Overview | Providers | `#overview/providers` | Inference providers | Provider bindings, availability, provenance, and configuration health | | Overview | Runtime | `#overview/runtime` | Runtime health | Local services, MCP connections, processes, and operational readiness | -| Overview | Intelligence | `#overview/intelligence` | Intelligence & learning | Machine-wide learning rollup across every ruflo-initialized project, plus near-live detail for one explicitly selected project | +| Overview | Intelligence | `#overview/intelligence` | Intelligence & learning | Machine-wide learning rollup across every project with memory or intelligence state, plus near-live detail for one explicitly selected project | | Usage | Scorecard | `#usage/score` | Usage scorecard | Token consumption, API-equivalent cost, efficiency, and trends | | Usage | Limits | `#usage/limits` | Provider limits | Current provider windows, reset timing, and available capacity | | Usage | Findings | `#usage/findings` | Usage findings | Actionable anomalies, efficiency opportunities, and evidence-backed recommendations | @@ -45,10 +45,12 @@ permanent. | Observability | Live | `#observability/live` | Observability · Live | Projects and roots with current presence or fresh meaningful activity | | Observability | History | `#observability/history` | Observability · History | Retained roots that are not currently Live | | System | Summary | `#system/summary` | Summary | Install size, retained data, live resource use, deployed inventory, and the machine's largest storage consumers in one glance | -| System | Storage | `#system/storage` | Storage | Where the retained bytes are, by category, host, project, and session — plus growth and advisory reclaimables in two safety tiers | -| System | Runtime | `#system/runtime` | Runtime | Live host processes, their CPU and memory, background daemons, and machine denominators | -| System | Catalog | `#system/catalog` | Catalog | Deduplicated skills, agents, commands, plugins, and MCP servers, with a per-host presence matrix | -| System | Projects | `#system/projects` | Projects | Projects ever seen across hosts vs still on disk; per on-disk project its approximate lines of code, detected stack, working-tree, `.git`, and `node_modules` size | +| System | Advisory | `#system/advisory` | Advisory | What could be reclaimed, in two safety tiers reported separately and never added — the only System area that suggests an action, and it still has no delete control | +| System | Sessions | `#system/sessions` | Sessions | The largest individual session files, the project each belongs to, and its share of that host's retained bytes | +| System | Storage | `#system/storage` | Storage | Where the retained bytes are, by category and host — learning stores counted separately because they dwarf everything else — plus per-series growth | +| System | Runtime | `#system/runtime` | Runtime | Live host processes, their CPU and memory, background daemons, and machine denominators — refreshed on the header's poll clock while open | +| System | Catalog | `#system/catalog` | Catalog | Every deduplicated skill, agent, command, plugin and MCP server across user scope and all projects on disk, with a per-host presence matrix and kind/host filters | +| System | Projects | `#system/projects` | Projects | Every repository with a remote that a host has recorded a session in — its approximate lines of code, language mix, total disk size and last activity. Worktrees, sub-folders and remote-less repositories are counted below the table, not listed | About is one scrolling page, so its hashes scroll to a section rather than swapping panels; `#about` alone opens the page at the top. @@ -111,7 +113,9 @@ Overview keeps status and routing in one health-first area: - **Providers** presents inference-provider bindings and their configuration provenance. - **Runtime** presents operational services, processes, and MCP readiness. - **Intelligence** presents memory, learning, and quality-improvement signals machine-wide: an - always-visible rollup folded across every ruflo-initialized project on this machine, plus detail + always-visible rollup folded across every project on this machine where memory or intelligence has + been activated — a `.claude-flow`, `.agentic-qe` or `.swarm` directory, whichever host created it + — plus detail for one explicitly selected, explicitly labeled project — the neural pattern store's current size, its separate lifetime patterns-learned counter, and reasoning-graph growth. Project selection defaults to whichever discovered project was most recently active; there is no implicit @@ -120,8 +124,32 @@ Overview keeps status and routing in one health-first area: ruflo/agentic-qe already write under `.claude-flow/` and updates near-live over a per-project SSE stream while the view is open, falling back to the general status poll otherwise. See [Project intelligence](ddd/project-intelligence.md) and - [ADR-0024](adr/0024-project-intelligence-telemetry.md) for the full model, the project-discovery - mechanism, and the two learning metrics' load-bearing distinction, now also at machine scope. + [ADR-0024](adr/0024-project-intelligence-telemetry.md) for the full model and the two learning + metrics' load-bearing distinction, and [ADR-0027](adr/0027-shared-project-census.md) for project + discovery. + +### Why project counts differ between tabs + +Every area derives its project list from one census, so a project means the same thing everywhere — +a session run in `myrepo/backend` belongs to `myrepo`, not to a project called `backend`, and an +agent worktree is not a peer of the repository it was cut from. + +The totals still differ, because the tabs ask different questions: + +| Tab | Counts | Over | +|---|---|---| +| Overview → Intelligence | projects with learning state | all time | +| Observability → History | projects with retained sessions | the selected history window | +| Usage → Scorecard | projects with recorded usage | the selected day window | +| System → Projects | **directories** ever seen, and the subset still on disk | all time | + +Two of those differences are structural rather than temporal. System counts **directories**, because +only a directory has bytes and lines to measure; the other tabs count **projects**. And a windowed +count is smaller than a lifetime one by exactly the projects you have not touched lately — a shorter +window, not a missing project. + +Each count carries the sentence explaining what it counted; on Intelligence it is behind +**how these projects were counted**, next to the rollup. ## Usage diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 0217f2c..568657b 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -231,12 +231,24 @@ Defaults (all overridable; your edits are marked `custom` and never re-seeded): |---|---|---| | specification, review, release | claude | `claude-sonnet-5` | | architecture, design, debugging, security-analysis | claude | `claude-opus-5` | -| implementation, testing, security-scan | codex | `gpt-5.4` | -| documentation, packaging | codex | `gpt-5.3-codex` | +| implementation, testing, security-scan | codex | `gpt-5.6-terra` | +| documentation, packaging | codex | `gpt-5.6-luna` | *(packaging & release are `ak`-added — ruflo ships templates for feature/security/refactor only.)* -**Known-good model choices** (verified 2026-07; any model your host CLI accepts also works): +**Retired codex models.** `gpt-5.4` and `gpt-5.4-mini` retire from Codex on **2026-08-31**, and +`gpt-5.3-codex` is already withdrawn for ChatGPT sign-in +([Codex models](https://developers.openai.com/codex/models)) — which is why the execution defaults +above moved to the 5.6 line. `ak` substitutes a retired model at read time, so no run dispatches to +one even if your `kit.json` still names it, and `ak sync` rewrites `seeded` routes that do. A `user` +pin is reported but never rewritten on disk (see +[ADR-0003](adr/0003-auto-seed-dual-host-provenance.md)). + +`claude-opus-4-8` is **not** retired — it carries no deprecation notice and stays pinnable. It is +merely no longer the default, which `ak status` reports as routing *divergence*: a trade for you to +weigh, cleared with `ak x host refresh` if you want the newer default. + +**Known-good model choices** (verified 2026-08; any model your host CLI accepts also works): > **Per-token price ≠ per-task cost.** A model that needs more agentic turns costs more > per task at the same per-token price. On subscription (`claude-code` oauth) billing the @@ -245,15 +257,14 @@ Defaults (all overridable; your edits are marked `custom` and never re-seeded): | Host | Model | When to use | |---|---|---| -| claude | `claude-opus-5` | new top Opus — same per-token price as 4.8, but ~2–3× the agentic turns on routine work; earns it at the hard end | +| claude | `claude-opus-5` | top Opus — the deepest reasoning, at ~2–3× the agentic turns of a balanced model on routine work; earns it at the hard end | | claude | `claude-sonnet-5` | near-Opus capability at a lower per-token price — review, spec, release | | claude | `claude-fable-5` | top capability (Mythos-class, above Opus 5) — hardest problems | | claude | `claude-haiku-4-5-20251001` | cheap/fast — high-volume mechanical work | -| claude | `claude-opus-4-8` | prior Opus generation — same per-token price, roughly half the turns on routine work | -| codex | `gpt-5.4` | coding + reasoning + agentic — recommended execution default | -| codex | `gpt-5.6-sol` | newest line; first-class max reasoning effort | -| codex | `gpt-5.3-codex` | pure coding-tuned — mechanical implementation & docs | -| codex | `gpt-5-codex-mini` | smallest/cheapest — escalation floor, high volume | +| claude | `claude-opus-4-8` | prior Opus generation — same per-token price as Opus 5, roughly half the agentic turns on routine work | +| codex | `gpt-5.6-sol` | flagship 5.6 — strongest on complex coding, computer use and security work; first-class max reasoning effort | +| codex | `gpt-5.6-terra` | balanced 5.6 — everyday implementation and testing at a materially lower per-token price than sol; the gpt-5.4 replacement | +| codex | `gpt-5.6-luna` | fastest/cheapest 5.6 — mechanical implementation, docs and packaging; the gpt-5.4-mini replacement | > **Where Opus 5 sits** ([announcement](https://www.anthropic.com/news/claude-opus-5), July 2026): > same $5/$25 per-Mtok pricing as Opus 4.8 with roughly double the Frontier-Bench diff --git a/docs/TRANSCRIPTS.md b/docs/TRANSCRIPTS.md index 1a8cb69..c94b5e0 100644 --- a/docs/TRANSCRIPTS.md +++ b/docs/TRANSCRIPTS.md @@ -36,12 +36,12 @@ rewritten; rule 3 of the module header, `usage-index.mjs:22-29`): | Host | Store | Discovered by | |---|---|---| -| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:733`) — exactly one level of project directories | -| Codex CLI | `~/.codex/sessions///
/rollout--.jsonl` | `listCodex` (`usage-index.mjs:705`) — the `yyyy/mm/dd` tree walk | +| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:786`) — exactly one level of project directories | +| Codex CLI | `~/.codex/sessions///
/rollout--.jsonl` | `listCodex` (`usage-index.mjs:758`) — the `yyyy/mm/dd` tree walk | -Roots come from `defaultRoots()` (`usage-index.mjs:697-701`) and are injectable +Roots come from `defaultRoots()` (`usage-index.mjs:750-754`) and are injectable for tests. A malformed line is skipped, never fatal (`jsonLines`, -`usage-index.mjs:328-334` — one corrupt line must not cost a whole file). +`usage-index.mjs:381-387` — one corrupt line must not cost a whole file). Host evidence is not inference-provider proof. A Claude transcript may describe Anthropic-, OpenRouter-, or Ollama-served inference. ADR-0016 defines separate @@ -52,34 +52,34 @@ transcript host/parser identity unless other evidence grounds the inference prov ### 1.1 Claude entry vocabulary Each line has a top-level `type`. The parser (`parseClaude`, -`usage-index.mjs:427-510`) reads: +`usage-index.mjs:480-563`) reads: | `type` | What the parser takes from it | |---|---| -| `ai-title` | The model-written session title (`usage-index.mjs:435`) — preferred over the first-prompt fallback | +| `ai-title` | The model-written session title (`usage-index.mjs:488`) — preferred over the first-prompt fallback | | `user` | A user-**role** turn — which is *not* the same as "the human"; see §3 | -| `assistant` | A model turn: `model` id, per-turn `usage` token counts, `tool_use` blocks (`usage-index.mjs:455-498`) | -| any | Side-band fields read regardless of type: `attributionSkill`/`attributionPlugin` (`usage-index.mjs:436-437`), `isSidechain` (`usage-index.mjs:438`), `cwd` for project derivation | +| `assistant` | A model turn: `model` id, per-turn `usage` token counts, `tool_use` blocks (`usage-index.mjs:508-551`) | +| any | Side-band fields read regardless of type: `attributionSkill`/`attributionPlugin` (`usage-index.mjs:489-490`), `isSidechain` (`usage-index.mjs:491`), `cwd` for project derivation | An assistant entry with `isApiErrorMessage: true` is a **local placeholder** Claude Code writes when a request dies before a real completion (connection drop, rate limit, auth failure — `model: ""`, all-zero usage). It is real engaged time but not a model attempt: counted as an *exception*, never -pushed into `models` or priced (`usage-index.mjs:498-517`; the full story is +pushed into `models` or priced (`usage-index.mjs:551-570`; the full story is [`USAGE-SCORECARD-METRICS.md`](USAGE-SCORECARD-METRICS.md) §10). ### 1.2 Codex entry vocabulary Codex rollout lines carry `type` + `payload`. The parser (`parseCodex`, -`usage-index.mjs:527-628`) reads: +`usage-index.mjs:580-681`) reads: | `type` / `payload.type` | What the parser takes from it | |---|---| -| `session_meta` | Authoritative session id, `cwd`, and `thread_source` (`usage-index.mjs:539-544`) — `"subagent"` marks a thread_spawn replay whose tokens are excluded from aggregation (`usage-index.mjs:609`; `USAGE-SCORECARD-METRICS.md` Appendix A, Bug B) | -| `turn_context` | The model id in effect from this point on (`usage-index.mjs:545`) | -| `event_msg` → `token_count` | A **cumulative** usage snapshot; only the last one is kept (`usage-index.mjs:609-611`) | -| `event_msg` → `user_message` | A real human prompt — Codex does not route tool output through this event (`usage-index.mjs:584-592`) | -| `event_msg` → `agent_message` | A model response (`usage-index.mjs:594-605`) | +| `session_meta` | Authoritative session id, `cwd`, and `thread_source` (`usage-index.mjs:592-597`) — `"subagent"` marks a thread_spawn replay whose tokens are excluded from aggregation (`usage-index.mjs:662`; `USAGE-SCORECARD-METRICS.md` Appendix A, Bug B) | +| `turn_context` | The model id in effect from this point on (`usage-index.mjs:598`) | +| `event_msg` → `token_count` | A **cumulative** usage snapshot; only the last one is kept (`usage-index.mjs:662-664`) | +| `event_msg` → `user_message` | A real human prompt — Codex does not route tool output through this event (`usage-index.mjs:637-645`) | +| `event_msg` → `agent_message` | A model response (`usage-index.mjs:647-658`) | Codex tool calls and tool outputs travel in event types the parser does not surface as turns at all — so a Codex transcript renders as a prompt/response @@ -95,8 +95,8 @@ The same parsers serve two very different callers, switched by `withTurns`: | Path | Entry point | `withTurns` | Message bodies | Cached? | |---|---|---|---|---| -| **Scan** — the aggregate index behind the Scorecard/Findings/Sessions views | `buildIndex` → `parseFile` (`usage-index.mjs:693`) | `false` | never held — holding them would balloon memory across 3,000+ files (`usage-index.mjs:437-440`) | yes: per-file derived records in `~/.config/agentic-kit/usage-index.json`, keyed `(path, mtime, size)`, invalidated wholesale by `SCHEMA_VERSION` (`usage-index.mjs:51`) | -| **Reader** — one transcript for the Transcript view | `readSession` (`usage-index.mjs:1310`) | `true` | full turn list built | **never** — every call re-reads and re-parses the one file | +| **Scan** — the aggregate index behind the Scorecard/Findings/Sessions views | `buildIndex` → `parseFile` (`usage-index.mjs:746`) | `false` | never held — holding them would balloon memory across 3,000+ files (`usage-index.mjs:490-493`) | yes: per-file derived records in `~/.config/agentic-kit/usage-index.json`, keyed `(path, mtime, size)`, invalidated wholesale by `SCHEMA_VERSION` (`usage-index.mjs:51`) | +| **Reader** — one transcript for the Transcript view | `readSession` (`usage-index.mjs:1363`) | `true` | full turn list built | **never** — every call re-reads and re-parses the one file | ![Figure: one parser, two read paths — the scan path (withTurns false) caches per-file records keyed by path, mtime and size; the reader path (withTurns true) builds full turns and is never cached](assets/transcript-read-paths.svg) @@ -117,11 +117,11 @@ recorded in `USAGE-SCORECARD-METRICS.md` Appendix A). |---|---|---| | `role` | all | `"user"` or `"assistant"` — the **Messages-API role**, not the author (see below) | | `at` | all | ISO timestamp | -| `text` | all | Flattened display text (`claudeText`, `usage-index.mjs:384` — binary payloads dropped: a pasted screenshot renders as `[image]`, a tool result is prefixed `[tool result]`) | -| `model` | assistant | The model id; the literal string `exception` for an API-error placeholder turn (`usage-index.mjs:473`) | +| `text` | all | Flattened display text (`claudeText`, `usage-index.mjs:437` — binary payloads dropped: a pasted screenshot renders as `[image]`, a tool result is prefixed `[tool result]`) | +| `model` | assistant | The model id; the literal string `exception` for an API-error placeholder turn (`usage-index.mjs:526`) | | `tools` | assistant | Tool names invoked in the turn | -| `prompt` | user | `isHumanPrompt`'s verdict (`usage-index.mjs:388-397`) — drives the **prompt counts** | -| `kind` | user | `'prompt'` \| `'tool-result'` \| `'context'` — drives the **attribution label** (`userTurnKind`, `usage-index.mjs:415-426`) | +| `prompt` | user | `isHumanPrompt`'s verdict (`usage-index.mjs:441-450`) — drives the **prompt counts** | +| `kind` | user | `'prompt'` \| `'tool-result'` \| `'context'` — drives the **attribution label** (`userTurnKind`, `usage-index.mjs:468-479`) | | `exception` | assistant | `true` on API-error placeholder turns | | `truncated`, `originalChars` | any | Present **only** when the turn was abridged (§4.3) | @@ -141,7 +141,7 @@ story is [Appendix A](#appendix-a--fix-history).) ### 3.2 `kind` — the attribution field -`userTurnKind` (`usage-index.mjs:470-475`) classifies every user-role turn: +`userTurnKind` (`usage-index.mjs:523-528`) classifies every user-role turn: | `kind` | Test | Meaning | |---|---|---| @@ -164,7 +164,7 @@ Two deliberate subtleties: - **`tool-result` outranks `context`**: a `tool_result` block on an `isMeta` entry is still tool feedback. -Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:657`) +Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:710`) — rollouts only record real prompts as `user_message` events (§1.2). Coverage: `tests/kit/usage-index.test.mjs` — "user-role turns carry a kind" @@ -175,18 +175,18 @@ and image-only pastes get the right kind" (the two edges). ## 4. The `readSession` pipeline — how one session becomes a payload -`readSession(id, opts)` (`usage-index.mjs:1297-1385`) is the only way +`readSession(id, opts)` (`usage-index.mjs:1350-1438`) is the only way transcript content leaves the module, and every step is a gate: ### 4.1 Locate, contain, bound 1. **Id grammar before any filesystem access** — `VALID_ID` (`/^[A-Za-z0-9._-]{1,128}$/`, `usage-index.mjs:83`) rejects traversal - shapes with `ERR_INVALID_SESSION_ID` (`usage-index.mjs:1329-1333`). -2. **Locate by id** across both roots (`locate`, `usage-index.mjs:1339`), + shapes with `ERR_INVALID_SESSION_ID` (`usage-index.mjs:1382-1386`). +2. **Locate by id** across both roots (`locate`, `usage-index.mjs:1392`), consulting the scan cache when present but never requiring it — `readSession` works with no prior `buildIndex`. -3. **Realpath containment** (`usage-index.mjs:1335-1349`) — the resolved file +3. **Realpath containment** (`usage-index.mjs:1388-1402`) — the resolved file must live under a transcript root *after* `realpathSync` collapses symlinks; a symlink planted inside a root pointing at `/etc/anything` passes a lexical `startsWith` but fails this. Roots are realpath'd too so @@ -198,8 +198,8 @@ transcript content leaves the module, and every step is a gate: ### 4.2 Parse and price The file is parsed with `withTurns: true` by the provider's parser -(`usage-index.mjs:1430-1437`), and `meta` is assembled -(`usage-index.mjs:1414-1442`) with the same fields the Sessions view rows +(`usage-index.mjs:1483-1490`), and `meta` is assembled +(`usage-index.mjs:1467-1495`) with the same fields the Sessions view rows carry — `prompts`, `responses`, `exceptions`, `sidechain`, `threadSource`, `models`, `tools`, `skill`/`plugin`, worktree — plus a `cost` priced from the same per-model usage rows `aggregate()` uses (the header used to render a @@ -211,7 +211,7 @@ Every turn body is passed through `maskSecrets` (`usage-index.mjs:196` — the 23 secret shapes) **server-side, before serialization**, then length-capped at `MAX_TURN_CHARS` (40,000, `usage-index.mjs:77`) with the marker appended -(`usage-index.mjs:1477-1487`). Two invariants: +(`usage-index.mjs:1530-1540`). Two invariants: - **Presence is the signal.** `truncated`/`originalChars` are emitted only when the slice fired, so a complete turn cannot be misread as abridged. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index f747b84..2f70aab 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -132,6 +132,21 @@ sources establish facts with observed, configured, inferred, or unknown provenan not silently create, adopt, or rewrite these bindings, and credentials remain environment-only. See [ADR-0016](adr/0016-capability-driven-integration-adapters.md). +## `ak system --json` fields removed in 4.0.0-alpha.41 + +Two fields left the runtime census. If you parse `ak system --json` (or `GET /api/system`), read +them defensively or drop them: + +| Removed | Where | Why | +|---|---|---| +| `runtime.daemons.budget` | daemon census | No local source exists for ruflo's launch budget — not circumstantially, structurally — so the field could only ever read `unknown`. A permanently unknowable quantity is removed rather than reported as degraded ([ADR-0023](adr/0023-fail-closed-operations-and-explicit-degradation.md) §9). `ruflo daemon budget` remains the way to ask. | +| `runtime.childProcessCount` | runtime census | Still counted by the process survey — it is what makes the per-host rows correct — but no longer republished. As a rendered figure it was a bare number with no denominator, no history and no action attached. | + +Nothing else was removed. `storage.topSessions` rows **gained** `projectLabel` and +`projectResolved`; the raw `project` key is unchanged. `catalog.items` now also covers +project-scoped `.claude/skills|agents|commands` across every project on disk, so the list is +longer — the shape is identical and deduplication by `(kind, name)` is unchanged. + ## QE-Court configs created before agentic-qe 3.13.3 `agentic-qe` 3.13.3 fixed its shipped QE-Court default and made configuration validation diff --git a/docs/USAGE-SCORECARD-METRICS.md b/docs/USAGE-SCORECARD-METRICS.md index 8ee9b9d..d27e0ec 100644 --- a/docs/USAGE-SCORECARD-METRICS.md +++ b/docs/USAGE-SCORECARD-METRICS.md @@ -64,8 +64,8 @@ schema change — `src/lib/usage-index.mjs:65`): | Claude Code | `~/.claude/projects//.jsonl` | one JSON object per line: `user`/`assistant` turns, each assistant turn carrying its own `usage` object | | Codex CLI | `~/.codex/sessions///
/rollout--.jsonl` | one JSON object per line: `session_meta`, `turn_context`, and `event_msg` records, the latter carrying **cumulative** `token_count` snapshots, not per-turn deltas | -The parsers are `parseClaude` (`usage-index.mjs:427-510`) and `parseCodex` -(`usage-index.mjs:527-628`). Both are pure functions over the raw file bytes — +The parsers are `parseClaude` (`usage-index.mjs:480-563`) and `parseCodex` +(`usage-index.mjs:580-681`). Both are pure functions over the raw file bytes — no network, no clock dependency beyond the transcript's own timestamps — so every downstream number traces back to bytes already on the user's disk. Nothing in this transcript pipeline calls a provider API or a billing endpoint; **no transcript @@ -128,14 +128,14 @@ responses = Σ over included sessions of session.responses **Source:** - Filter: a parsed record with zero assistant turns is dropped entirely — "no - assistant turn → not a session" (`usage-index.mjs:935`) — and a record whose + assistant turn → not a session" (`usage-index.mjs:988`) — and a record whose last activity falls outside the requested window is dropped too - (`usage-index.mjs:936`). + (`usage-index.mjs:989`). - `responses` accumulation: Claude increments per assistant message - (`usage-index.mjs:504-509`); Codex increments per `agent_message` event - (`usage-index.mjs:651-655`). + (`usage-index.mjs:557-562`); Codex increments per `agent_message` event + (`usage-index.mjs:704-708`). - Totals: `totals.responses += s.responses` per included session - (`usage-index.mjs:884`). + (`usage-index.mjs:937`). - Render: `kpi("sessions", fmtNum(t.sessions), fmtNum(t.responses)+" assistant turns", "")` (`dashboard/client.mjs`). @@ -174,7 +174,7 @@ cost = (inputUnits × rate_in + output × rate_out) / 1,000,000 summed across every row in the window. -**Source:** `costOf()`, `src/lib/pricing.mjs:247-253`, reproduced verbatim: +**Source:** `costOf()`, `src/lib/pricing.mjs:253-259`, reproduced verbatim: ```js export function costOf(usage) { @@ -188,17 +188,17 @@ export function costOf(usage) { ``` `CACHE_READ_MULTIPLIER = 0.1`, `CACHE_WRITE_MULTIPLIER = 1.25` -(`pricing.mjs:154-155`) — see §13 for why these two numbers are correct for +(`pricing.mjs:160-161`) — see §13 for why these two numbers are correct for **both** Anthropic and OpenAI, which is why `costOf` needs no per-provider branch on the multiplier (only on the base `rate_in`/`rate_out`, resolved by -`priceFor`, `pricing.mjs:214-227`). +`priceFor`, `pricing.mjs:220-233`). Rate resolution is **longest-prefix match** on a normalized model id -(`pricing.mjs:163-172`), so a dated release (`claude-haiku-4-5-20251001`) +(`pricing.mjs:169-178`), so a dated release (`claude-haiku-4-5-20251001`) resolves to the same entry as its bare alias, and a more specific entry (`gpt-5.6-sol`) is never shadowed by a shorter one (`gpt-5.6`). An id matching nothing gets `FALLBACK_PRICE` — Sonnet-class rate, `$3`/`$15` -(`pricing.mjs:151`) — rather than `$0`, so an unrecognized model can never be +(`pricing.mjs:157`) — rather than `$0`, so an unrecognized model can never be silently free; `matched: false` travels with the result so a maintainer can find fallback-priced rows if the table needs a new entry. @@ -208,12 +208,12 @@ Each table entry is a **schedule** — an ordered list of periods, each with the day it takes effect. Nearly every entry has exactly one period that has always applied (`anthropic(5, 25)` builds that shape); an entry whose rate the vendor has *published* a change to carries more than one (`schedule`, -`pricing.mjs:41-55`). `periodOn` (`pricing.mjs:187`) picks the last period +`pricing.mjs:41-55`). `periodOn` (`pricing.mjs:193`) picks the last period already in effect on the given day, comparing ISO date strings lexicographically so no `Date` parsing is involved and the module stays clock-free. -`aggregate()` passes each usage row's own `day` (`usage-index.mjs:926`), which +`aggregate()` passes each usage row's own `day` (`usage-index.mjs:979`), which it already has because rows are keyed by `(day, model)`. **This is the whole point:** tokens metered in August must still read as August's rate when the panel is opened in December. Pricing by *today's* date instead would restate a @@ -235,7 +235,7 @@ Two rules bound the mechanism: Codex promo would be a one-line edit rather than new machinery. Rates that vary by *how* a request was served — regional uplift, large-prompt surcharge, service tiers — are a different axis, are deliberately **not** expressible - here, and remain in `UNMODELLED_PRICING_FACTORS` (`pricing.mjs:142-144`) + here, and remain in `UNMODELLED_PRICING_FACTORS` (`pricing.mjs:148-150`) because a transcript does not record the endpoint or tier. A `priceFor` call with no day prices as of `PRICES_AS_OF`, the table's @@ -295,9 +295,9 @@ tokens = input + output + cacheRead + cacheWrite (summed across all rows in wi ``` **Source:** `t.tokens` from `totals`, accumulated per row at -`usage-index.mjs:935` (`rowTokens = row.input + row.output + row.cacheRead + +`usage-index.mjs:988` (`rowTokens = row.input + row.output + row.cacheRead + row.cacheWrite`) and rolled into `totals.tokens` via `addTo` -(`usage-index.mjs:1027-1033`). Rendered with `fmtTok()` +(`usage-index.mjs:1080-1086`). Rendered with `fmtTok()` (`dashboard/client.mjs`): `≥1e9` → `"X.XB"`, `≥1e6` → `"X.XM"`, `≥1e3` → `"X.XK"`, else the rounded integer. @@ -309,9 +309,9 @@ numbers as percentages of `t.tokens` (`dashboard/client.mjs`, **What "input" excludes.** For both providers, the `input` counter recorded per row is **gross input minus cached input** — Claude's parser reads `cache_read_input_tokens` and `cache_creation_input_tokens` as separate fields -the provider already reports separately (`usage-index.mjs:545-546`); Codex's +the provider already reports separately (`usage-index.mjs:598-599`); Codex's parser subtracts `cached_input_tokens` from `input_tokens` explicitly -(`usage-index.mjs:666-675`, `input: Math.max(0, gross - cacheRead)`) because +(`usage-index.mjs:719-728`, `input: Math.max(0, gross - cacheRead)`) because Codex's own `input_tokens` field **includes** cached tokens and would double-count them against the separately-reported `cacheRead` figure if left as-is. This is asserted by test: @@ -405,15 +405,15 @@ session data, and each needs its own fix: sorts intervals and merges any two that overlap **or exactly touch** (`s <= curEnd`, `usage-index.mjs:105`), returning total covered seconds rounded to the nearest second. -- `activeIntervals()` (`usage-index.mjs:352-363`) — splits one session's +- `activeIntervals()` (`usage-index.mjs:405-416`) — splits one session's sorted timestamp list into sub-intervals wherever a gap exceeds `IDLE_GAP_MS`; "a run of one timestamp yields a zero-length interval and so - contributes nothing" (comment, `usage-index.mjs:346-350`). + contributes nothing" (comment, `usage-index.mjs:399-403`). - Aggregation: `totals.engagedSeconds = mergeIntervals(sessions.flatMap(s => - s._active))` (`usage-index.mjs:927`); `totals.spanUnionSeconds = - mergeIntervals(sessions.map(s => s._span))` (`usage-index.mjs:926`); + s._active))` (`usage-index.mjs:980`); `totals.spanUnionSeconds = + mergeIntervals(sessions.map(s => s._span))` (`usage-index.mjs:979`); `totals.spanMinutes` is a running sum of `s._span[1] - s._span[0]` across - the loop (`usage-index.mjs:888`, finalized `usage-index.mjs:925`). + the loop (`usage-index.mjs:941`, finalized `usage-index.mjs:978`). - Render: `fmtHours()` (`dashboard/client.mjs`, `≥10h` rounds to the nearest hour, else one decimal place) and `fmtMins()` (`dashboard/client.mjs`, `≥60min` rounds to hours, else whole @@ -462,12 +462,12 @@ byDay[day].cost = Σ costOf(row) for every usage row whose day == that key **Source:** the day key is the row's own `row.day`, computed once at parse time as **local calendar day**, not UTC -(`usage-index.mjs:542`/`usage-index.mjs:679` call `localDay(at)`) — so a +(`usage-index.mjs:595`/`usage-index.mjs:732` call `localDay(at)`) — so a session that runs from 23:58 local to 00:05 local is billed to the day its *first* row landed on (test: `tests/kit/usage-index.test.mjs:634`, "a session that opens before midnight is counted on its first billed day"). Accumulation: -`byDay[row.day].cost += rowCost` (`usage-index.mjs:846`). Bar height: +`byDay[row.day].cost += rowCost` (`usage-index.mjs:899`). Bar height: `h = maxDay ? max(2, cost/maxDay*100) : 2` (`dashboard/client.mjs`) — every non-empty day gets a visually nonzero bar (floor of 2%), so a very cheap day is never rendered as invisible. @@ -488,11 +488,11 @@ renders "no sessions in window" instead of zeroed figures (`dashboard/client.mjs`). **Formula:** identical aggregation to every other bucket -(`byProvider[s.provider]`, populated via `addTo()`, `usage-index.mjs:889-898`, - called once per session at `usage-index.mjs:1031`), keyed by the literal string +(`byProvider[s.provider]`, populated via `addTo()`, `usage-index.mjs:942-951`, + called once per session at `usage-index.mjs:1084`), keyed by the literal string `"claude"` or `"codex"` assigned at parse time (`blankSession(id, 'claude')` / `blankSession(id, 'codex')`, -`usage-index.mjs:345-355`, `parseClaude`/`parseCodex` entry points). +`usage-index.mjs:398-408`, `parseClaude`/`parseCodex` entry points). **Why this pairing is the one under the most scrutiny.** Both providers' tokens are summed into the *same* `tokens`/`cost` fields using the *same* @@ -528,9 +528,9 @@ punchcard[dow + "-" + hour] += 1 per assistant/agent_message response, at its ``` **Source:** incremented once per Claude assistant turn -(`usage-index.mjs:504-509`, keyed by `punchKey(at)`) and once per Codex -`agent_message` (`usage-index.mjs:651-655`), merged into the window-level -`punchcard` object per session (`usage-index.mjs:1000`). Cell intensity is +(`usage-index.mjs:557-562`, keyed by `punchKey(at)`) and once per Codex +`agent_message` (`usage-index.mjs:704-708`), merged into the window-level +`punchcard` object per session (`usage-index.mjs:1053`). Cell intensity is linear against the single busiest cell in the window: `v = pcMax ? n/pcMax : 0` (`dashboard/client.mjs`) — this is a **relative**, not absolute, scale, so the heatmap's brightest cell is always @@ -563,9 +563,9 @@ byModel[model].sessions = count of DISTINCT sessions whose s.models includes th ``` **Source:** cost/tokens/responses accumulate inside the usage-row loop -(`usage-index.mjs:896-922`); the `sessions` count is deliberately computed +(`usage-index.mjs:949-975`); the `sessions` count is deliberately computed **separately**, once per session over its `s.models` array -(`usage-index.mjs:898-903`) rather than inside the cost loop, precisely +(`usage-index.mjs:951-956`) rather than inside the cost loop, precisely **so that a model can appear in `byModel` — with a nonzero session count — even in a session that contributed zero cost/tokens/responses for that model.** This is not an edge case invented for this document: it is the @@ -575,11 +575,11 @@ excluded subagent-replay session still shows up as "used," at zero cost, rather than vanishing. `byModel[...].responses` is populated from `row.responses` -(`usage-index.mjs:964`), which in turn comes from the `responses` field +(`usage-index.mjs:1017`), which in turn comes from the `responses` field passed into `addUsage()` at the call site — `1` per Claude assistant turn -(`usage-index.mjs:484-490`), or `rec.responses` (the session's whole response +(`usage-index.mjs:537-543`), or `rec.responses` (the session's whole response count) once per Codex session, passed at the single point Codex calls -`addUsage` (`usage-index.mjs:613-624`). +`addUsage` (`usage-index.mjs:732-738`). **Render:** `bar(name, fmtUsd(cost), fmtTok(tokens)+" · "+fmtNum(responses)+" resp", pct(cost, topModelCost), false)` (`dashboard/client.mjs`), @@ -595,14 +595,14 @@ split `server_error` 27, `authentication_failed` 3, `rate_limit` 3 — three distinct underlying causes, one placeholder shape). The parser branches on `isApiErrorMessage === true` -(`usage-index.mjs:469-478`): the turn still increments `rec.responses` -and the punchcard (`usage-index.mjs:457-460`) — it *is* real engaged +(`usage-index.mjs:522-531`): the turn still increments `rec.responses` +and the punchcard (`usage-index.mjs:510-513`) — it *is* real engaged time, someone was genuinely waiting on it — but it is never pushed into `rec.models` and `addUsage()` is never called for it, so it can no longer create a `byModel` row of any kind. It increments a separate -`rec.exceptions` counter instead (`usage-index.mjs:470`), rolled up into -`totals.exceptions` (`usage-index.mjs:995-999`) and surfaced per-session -(`usage-index.mjs:920-934`, alongside the existing `sidechain`/`threadSource` +`rec.exceptions` counter instead (`usage-index.mjs:523`), rolled up into +`totals.exceptions` (`usage-index.mjs:1048-1052`) and surfaced per-session +(`usage-index.mjs:973-987`, alongside the existing `sidechain`/`threadSource` flags — inspectable in the Sessions tab, never hidden). When `totals.exceptions > 0`, the panel header shows a small `"· N dropped/errored turns excluded"` note (`dashboard/client.mjs`); @@ -756,7 +756,7 @@ own published table, not a transcription from a secondary source: | Claude Sonnet 4.6 / 4.5 | $3/MTok | $3.75/MTok | $6/MTok | $0.30/MTok | $15/MTok | | Claude Haiku 4.5 | $1/MTok | $1.25/MTok | $2/MTok | $0.10/MTok | $5/MTok | -Every value in `pricing.mjs`'s Anthropic entries (`pricing.mjs:74-108`) matches +Every value in `pricing.mjs`'s Anthropic entries (`pricing.mjs:74-114`) matches this table's "Base input" and "Output" columns exactly. Note that the two Sonnet 5 rows above are not a documentation convenience — they are exactly what the code encodes, as the two periods of that entry's schedule (§3a), so the table @@ -777,7 +777,7 @@ Anthropic's own prompt-caching documentation **[C2]** states the multipliers in prose, independent of the pricing table: *"5-minute cache write tokens are 1.25 times the base input tokens price... Cache read tokens are 0.1 times the base input tokens price."* This is the second, independent confirmation of -`CACHE_READ_MULTIPLIER`/`CACHE_WRITE_MULTIPLIER` (`pricing.mjs:154-155`). +`CACHE_READ_MULTIPLIER`/`CACHE_WRITE_MULTIPLIER` (`pricing.mjs:160-161`). ### 13.2 OpenAI (Codex) — hand-maintained, no canonical machine-readable source @@ -826,8 +826,8 @@ hand-maintained and drift-prone, not vendor-confirmed via automated fetch. ### 13.3 What the pricing table deliberately does not model -Recorded verbatim from `pricing.mjs:125-141` (`UNMODELLED_PRICING_FACTORS`, -`pricing.mjs:142-144`) because listing known gaps is what makes the +Recorded verbatim from `pricing.mjs:131-147` (`UNMODELLED_PRICING_FACTORS`, +`pricing.mjs:148-150`) because listing known gaps is what makes the *modelled* factors credible: - **Regional-processing uplift.** OpenAI charges +10% on data-residency @@ -886,7 +886,7 @@ both credential-free for ak: `windowDurationMins: 10080` (the weekly). Windows are therefore keyed and labelled by duration (`windowLabel`, `quota.mjs:44`), never by slot name. The same rule applies to the historical snapshots parsed out of rollouts: the -normalizer at `usage-index.mjs:591-615` keeps a flat `windows` list keyed by +normalizer at `usage-index.mjs:644-668` keeps a flat `windows` list keyed by `window_minutes`. **Freshness is part of the number.** Both sides carry `fetchedAt`; the view @@ -911,13 +911,13 @@ Codex ≥0.140 maintains its own SQLite thread ledger (`~/.codex/state_N.sqlite` — the `N` is a migration generation, so `codexStateDb` (`codex-state.mjs:30`) globs and takes the newest). `readCodexState` (`:49`) reads per-thread `thread_source` (`user` vs `subagent`) plus `thread_spawn_edges`, and -`applyCodexLedger` (`usage-index.mjs:1264-1275`) overlays that onto parsed +`applyCodexLedger` (`usage-index.mjs:1317-1328`) overlays that onto parsed sessions: a ledger-identified subagent has its token usage stripped — its rollout replays the parent's entire token history (ccusage/ccusage#950 measured up to 91× inflation) — while the session record stays visible. The rollout's own `session_meta.thread_source` sniff remains as the fallback when the ledger is absent or migrated beyond recognition. Codex sessions also carry -`reasoningOutput` (`usage-index.mjs:686-687`) — reasoning tokens are a **subset** +`reasoningOutput` (`usage-index.mjs:739-740`) — reasoning tokens are a **subset** of output tokens and are annotation only, never added to any sum. ## 14. Known limitations, restated as a single checklist @@ -962,14 +962,14 @@ commit `540be18` on this branch. `parseCodex`'s single `addUsage()` call never included a `responses` field — Claude's parser passes `responses: 1` per assistant turn -(`usage-index.mjs:503`, the current equivalent), but Codex's call +(`usage-index.mjs:556`, the current equivalent), but Codex's call passed no such field at all. Because `byModel[model].responses` is summed -directly from each usage row's `responses` field (`usage-index.mjs:921`, +directly from each usage row's `responses` field (`usage-index.mjs:974`, `m.responses += row.responses`), **every** Codex model in §10's "Models in Play" list displayed `0 resp` regardless of real token/cost volume or actual `agent_message` count. **Fix:** `parseCodex` now passes `responses: rec.responses` (the session's own tallied response count, -`usage-index.mjs:654`) on its `addUsage()` call. +`usage-index.mjs:707`) on its `addUsage()` call. #### Bug B — subagent thread-replay could double-bill tokens @@ -989,13 +989,13 @@ at face value (correctly avoiding the separate naive-summing bug **[C5]** documents, since it already used last-event-only logic — see §4's worked example) but performed **no de-duplication** against a parent session a subagent file might be replaying. **Fix:** the parser now reads -`session_meta.thread_source` (`usage-index.mjs:575-579`, confirmed as a real +`session_meta.thread_source` (`usage-index.mjs:628-632`, confirmed as a real Codex rollout field by **[C7]**) and skips the `addUsage()` call entirely -when its value is `'subagent'` (`usage-index.mjs:609`, guard condition +when its value is `'subagent'` (`usage-index.mjs:662`, guard condition `rec.threadSource !== 'subagent'`). The session record itself is **not** dropped — it remains visible in the Sessions tab with `threadSource` surfaced (mirroring the existing `sidechain` flag Claude sessions already -carry, `usage-index.mjs:294`), so a maintainer auditing the raw data can +carry, `usage-index.mjs:347`), so a maintainer auditing the raw data can still see it; it simply contributes zero tokens/cost, exactly as intended by the "models still shows up in §10's list, with zero cost" mechanism §10 describes. diff --git a/docs/adr/0002-activity-vocabulary-defaults-from-ruv-templates.md b/docs/adr/0002-activity-vocabulary-defaults-from-ruv-templates.md index 959e09c..eac846f 100644 --- a/docs/adr/0002-activity-vocabulary-defaults-from-ruv-templates.md +++ b/docs/adr/0002-activity-vocabulary-defaults-from-ruv-templates.md @@ -38,6 +38,13 @@ as such** wherever surfaced (an `ak` tag in the UI, a comment in the defaults ta Default *models* map to the host's appropriate tier (Opus for deep reasoning, Sonnet for review, a Codex model for execution) and are treated as **soft defaults** — see the "open question" on pinning live model IDs. +Tier is the pairing key, not the model id: `MODEL_CATALOG` spells `flagship`/`balanced`/`fast` +identically on both hosts so primary-host mirroring can map a route to its counterpart's equivalent. +As of 2026-08-07 execution routes to `gpt-5.6-terra` (balanced) and mechanical work to `gpt-5.6-luna` +(fast), following OpenAI's own migration off `gpt-5.4`/`gpt-5.4-mini` before their 2026-08-31 Codex +retirement; deep reasoning routes to `claude-opus-5`. Withdrawn ids are handled by the retirement +mechanism in [ADR-0003](0003-auto-seed-dual-host-provenance.md), not by editing this table alone. + ## Consequences - Defaults are **provably grounded** and explainable ("architect→claude because `featureDevelopment` does"). diff --git a/docs/adr/0003-auto-seed-dual-host-provenance.md b/docs/adr/0003-auto-seed-dual-host-provenance.md index 868888d..53bc3ec 100644 --- a/docs/adr/0003-auto-seed-dual-host-provenance.md +++ b/docs/adr/0003-auto-seed-dual-host-provenance.md @@ -2,9 +2,9 @@ - **Status:** Amended by [ADR-0020](0020-ga-stable-surfaces.md) - **Date:** 2026-07-23 -- **Updated:** 2026-07-30 +- **Updated:** 2026-08-07 - **Update note:** Preserved subscription-safe seeding and provenance while moving intent to the - canonical routing envelope. + canonical routing envelope; separated model **retirement** from route **divergence** (2026-08-07). - **Deciders:** agentic-kit maintainers > **GA amendment:** subscription-safe seeding and user-intent preservation remain. The persisted @@ -40,6 +40,34 @@ Every `ActivityRoute` carries **provenance**: `source: 'default' | 'seeded' | 'u **never clobbered**. `ak sync` reasserts the policy but never re-seeds or overwrites `user` routes. `ak x provider off` clears the whole policy and reverts projections. +### Amendment (2026-08-07) — retired models are not divergence + +Provenance answers "may ak change this value?". It does **not** answer "does this value still work?", +and one case needs both: a model the host has **withdrawn**. A route naming one is not a stale +preference, it is a scheduled hard failure — `gpt-5.4` and `gpt-5.4-mini` stop answering in Codex on +2026-08-31, and `gpt-5.3-codex` already has. + +So retirement is separated from divergence, and the two are handled differently: + +| | divergence (`divergedRoutes`) | retirement (`RETIRED_MODELS`) | +|---|---|---| +| what happened | the defaults moved | the model stops answering | +| is there a trade? | yes — a newer default can cost 2–3× the agentic turns | no | +| reported as | a choice, with both models' cost-per-task notes | a fact, with the retirement date | +| cleared by | an explicit `ak x host refresh` | `ak sync`, automatically | +| touches a `user` pin? | never | **substituted at read time, never rewritten on disk** | + +That last row is the only place ak overrides deliberate user intent, and it is deliberate: honoring a +pin into a model that no longer exists fails the run rather than respecting the pin. The override is +confined to `resolveRoutes()` — the read boundary every dispatch path already goes through — so the +file on disk still says what the user wrote, and every surface that shows the substituted model also +names the id it replaced (`retiredFrom`). + +Because this power is easy to misuse, `RETIRED_MODELS` admits **only** models with a published +withdrawal notice from the host, cited in the map. "We would rather they used the newer one" is +divergence, not retirement — `claude-opus-4-8` is the worked example: superseded as a default, no +deprecation notice, therefore deliberately absent from the map and still pinnable. + ## Consequences - **Zero-step** onboarding for dual-host users; **no cost surprise** (subscription/local only). @@ -47,9 +75,16 @@ Every `ActivityRoute` carries **provenance**: `source: 'default' | 'seeded' | 'u - Single-host users are unaffected — nothing is seeded, projections stay empty, behavior is unchanged. - Requires tracking `source` on each route and honoring it in every write path (seed, pick, sync, refresh). - If a route later points at a now-disabled host, `ak status` warns rather than silently mutating it. +- A retired model can never be dispatched, whatever the policy on disk says — but the substitution is + invisible unless surfaces render `retiredFrom`, so every route-rendering surface must carry it. +- `RETIRED_MODELS` needs maintaining against host deprecation notices. A missed entry degrades to a + failed run with a clear upstream error; a *wrong* entry silently ignores a user's pin, which is far + worse — hence the citation requirement. ## References -- `src/lib/providers.mjs` `bothHostsEnabled`, `_managedBy` / `.bak` discipline (`applyAqeRouter`, - `undoAqeRouter`, `writeJsonWithBackup`) +- `src/lib/routing.mjs` `RETIRED_MODELS`, `retirementOf`, `migrateRetiredRoutes`, `resolveRoutes` +- `src/lib/providers.mjs` `bothHostsEnabled`, `migrateRetiredRoutesInConfig`, `_managedBy` / `.bak` + discipline (`applyAqeRouter`, `undoAqeRouter`, `writeJsonWithBackup`) +- `tests/kit/routing-retirement.test.mjs` - ADR-0001, ADR-0002 diff --git a/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md b/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md index 5a9fedf..d81d674 100644 --- a/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md +++ b/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md @@ -1,6 +1,7 @@ # ADR-0023 — Fail-closed mutations and explicit degraded operation evidence - **Status:** Implemented +- **Updated:** 2026-08-07 — §9 permanently unmeasurable quantities, §10 stated exclusions - **Date:** 2026-08-04 - **Updated:** 2026-08-06 - **Update note:** Generalized setup preflight into a required host-adapter trust contract, added @@ -173,9 +174,34 @@ PATH before loading or launching the CLI. The ordinary matrix runs these tests o Windows. The scheduled/manual nightly additionally packs the current artifact and performs real setup on `macos-latest` with all global packages and user/project files under `runner.temp`. +### 9. A permanently unmeasurable quantity is deleted, not degraded (2026-08-07) + +The rules above cover a quantity that *could* be measured and was not: it renders as unknown with a +reason. They did not cover a quantity that no code path can *ever* produce. The System area shipped +one — an "AI-worker budget" tile hardcoded to unknown with the reason "ruflo exposes no local budget +state this collector can read". It could never render anything else. + +An always-unknown tile is not honest degradation, it is a promise the product cannot keep, occupying +space and teaching the reader to ignore unknowns. So: a field whose reason is *structural* rather +than circumstantial is **removed from the collector, the payload and the panel**, not rendered as +degraded. The distinction is whether a plausible future state of the machine would populate it. If +one would, it stays and degrades; if none would, it was never a measurement. + +### 10. An excluded figure is still stated (2026-08-07) + +Storage lifts learning stores out of its shared charts because at ~99% of retained bytes they flatten +every other series. Exclusion for legibility is allowed; *silent* exclusion is not. The excluded +category is reported on its own card, the charts say what they leave out, and the per-host split +names the non-host bytes it drops with their figure — so the bars still account for the whole of the +donut beside them. A reader must never have to reconcile two panels and find the difference +unexplained. + ## Consequences - A fallback can keep work available without being mislabeled healthy. +- An unknown carries information, because nothing that is permanently unknowable is rendered as one. +- A chart may exclude a category for legibility, but the panel says so and the excluded figure is + still reachable. - Temporary SQLite locks and corrupt stores no longer become measured zero usage. - Backup-path problems stop setup/sync instead of destroying the only recoverable pre-write state. - Status-line failures stay silent for ordinary users and become diagnosable without logging content. diff --git a/docs/adr/0024-project-intelligence-telemetry.md b/docs/adr/0024-project-intelligence-telemetry.md index 3f2e41b..45120a8 100644 --- a/docs/adr/0024-project-intelligence-telemetry.md +++ b/docs/adr/0024-project-intelligence-telemetry.md @@ -1,8 +1,8 @@ # ADR-0024 — Project intelligence: live learning telemetry from ruflo/agentic-qe's own state -- **Status:** Implemented +- **Status:** Implemented; discovery amended by [ADR-0027](0027-shared-project-census.md) - **Date:** 2026-08-05 -- **Updated:** 2026-08-05 +- **Updated:** 2026-08-07 - **Update note:** Extended Intelligence from one project's telemetry, implicitly tied to the dashboard server's own launching cwd, to a machine-wide catalog of every ruflo-initialized project plus an explicitly selected, explicitly labeled detail project (defaulting to @@ -13,9 +13,22 @@ - **Deciders:** agentic-kit maintainers - **Related:** [ADR-0005](0005-dashboard-in-page-routing-reveal.md), [ADR-0009](0009-usage-scorecard-local-transcript-analytics.md), - [ADR-0012](0012-observability.md) + [ADR-0012](0012-observability.md), + [ADR-0027](0027-shared-project-census.md) -**2026-08-05 machine-wide discovery amendment:** `src/lib/dashboard/project-discovery.mjs` adds +> **Superseded 2026-08-07 by [ADR-0027](0027-shared-project-census.md).** The machine-wide +> discovery amendment below is retained as the record of how discovery worked from 2026-08-05; its +> mechanism is gone. `discoverRuvfloProjects()` and `project-discovery.mjs` are retired, and +> `registryWorkspaces()` is module-private again. Two things it got wrong, both visible only at +> scale: requiring `.claude-flow/neural/` answered "has ruflo *trained* here" rather than "is +> memory or intelligence active here", excluding agentic-qe and swarm state and every non-Claude +> host; and the 150-transcript bound made discovery a function of recency. On the machine this +> amendment was verified against, it found 4 projects where 17 had learning state. Project +> discovery now comes from the shared census, which lists directories and folds them onto project +> identity. Everything else in this ADR — the intel payload, the SSE pool, the panel itself — +> stands unchanged. + +**2026-08-05 machine-wide discovery amendment (superseded):** `src/lib/dashboard/project-discovery.mjs` adds `discoverRuvfloProjects()`, unioning three sources into one deduplicated, most-recently-active-first catalog of every project on this machine ruflo has genuinely initialized — a `.claude-flow/neural/` subdirectory present, not merely a bare `.claude-flow/`. Source 1 reuses `registryWorkspaces()` from @@ -237,7 +250,7 @@ path. - `src/lib/dashboard/intel-history.mjs` (`readIntelHistory`, `readMachineWideIntel`), `tests/kit/intel-history.test.mjs` -- `src/lib/dashboard/project-discovery.mjs` (`discoverRuvfloProjects`), +- `src/lib/project-census.mjs` (`projectCensus`, `projectsInScope`) — replaced `src/lib/dashboard/project-discovery.mjs` (`discoverRuvfloProjects`) per ADR-0027, `tests/kit/project-discovery.test.mjs` - `src/lib/live/intelligence-watch.mjs`, `tests/kit/intelligence-watch.test.mjs` - `src/lib/dashboard-server.mjs` (`collectData`, `buildProjectSnapshotCache`, diff --git a/docs/adr/0025-machine-footprint-metrics.md b/docs/adr/0025-machine-footprint-metrics.md index 819a3e8..a8cd668 100644 --- a/docs/adr/0025-machine-footprint-metrics.md +++ b/docs/adr/0025-machine-footprint-metrics.md @@ -16,6 +16,24 @@ [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md), [ADR-0024](0024-project-intelligence-telemetry.md) +> **2026-08-07 amendment.** Four changes, each recorded where it belongs: +> project discovery moved to the shared census ([ADR-0027](0027-shared-project-census.md)); +> Storage reports learning stores on their own card and its per-host split covers real hosts only, +> naming the excluded figures rather than dropping them ([ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md) §10); +> the Runtime census dropped its launch-budget and child-process fields, the first because no code +> path could ever populate it ([ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md) §9); +> and the Catalog now covers project scope across every project on disk, not just the launching repo. +> The Projects table dropped its forge sub-line, its presence-only stack chips and its +> tree/.git/node_modules breakdown from the RENDERING — all three are still measured and still ship +> on `ak system --json` — and now lists only repositories with a remote that a host has recorded a +> session in, counting the excluded directories beneath it. +> +> The area also grew from five sub-views to seven: **Advisory** and **Sessions** split out of +> Storage. Advisory earns its own tab because it is the only part of System that suggests an +> action while every other part reports what is; that distinction was invisible while it sat as a +> card under a byte chart. Neither split changes a measurement, and Advisory still has no delete +> verb (§6 stands). + ## Context The dashboard currently answers three families of questions, each owned by its own context: @@ -111,7 +129,7 @@ Storage Breakdown tree: category → host → project → session. Transcripts learning stores vs kit caches. Trailing-30d growth sparkline per host. Top-N largest sessions/files. Advisory reclaimable candidates in two safety tiers, never one total. Runtime Live process table (host, pid, CPU%, RSS, uptime, bound project) + combined totals. - Daemon census (count, age vs TTL, budget state). Child/MCP server process count. + Daemon census (count, age vs TTL). Catalog Deduplicated skills / agents / commands / plugins / MCP servers, each with a per-host presence matrix (which hosts carry it). Projects Table: project (name links to its git remote's web page when one exists — derived @@ -152,7 +170,7 @@ Metrics marked ✚ are additions beyond the requesting examples; the taxonomy is | Install | Shared caches: npx cache envs, brain KB, browser binaries ✚ | known roots | | Install | Total install bytes + machine free-space denominator ✚ | walk + `statfs` | | Runtime | Per live host process: pid, host, CPU%, RSS, uptime, bound project | existing runtime survey + `ps -o pcpu,rss` | -| Runtime | Daemon census: count, age vs 12h TTL, budget state ✚ | existing daemon registry | +| Runtime | Daemon census: count, age vs 12h TTL ✚ | existing daemon registry | | Runtime | Child / MCP-server process count ✚ | survey process tree | | Storage | Transcript bytes + file counts: host → project → session | transcript-root walk | | Storage | Host ledgers/logs: Codex `state_N.sqlite`, statusline tee, runtime-debug log, OpenCode store | known paths | diff --git a/docs/adr/0027-shared-project-census.md b/docs/adr/0027-shared-project-census.md new file mode 100644 index 0000000..a62d775 --- /dev/null +++ b/docs/adr/0027-shared-project-census.md @@ -0,0 +1,105 @@ +# ADR-0027 — One project census, four scopes, every count explains itself + +- **Status:** Implemented +- **Date:** 2026-08-07 +- **Deciders:** agentic-kit maintainers +- **Related:** [ADR-0012](0012-observability.md), + [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md), + [ADR-0024](0024-project-intelligence-telemetry.md), + [ADR-0025](0025-machine-footprint-metrics.md) +- **Supersedes:** the machine-wide discovery amendment in ADR-0024 (`discoverRuvfloProjects`) + +## Context + +Four dashboard areas answered "what projects does this machine have" four different ways, from +four different sources, with four different naming rules — and reported four different numbers for +the same machine: + +| Area | Source of truth | Naming | Reported | +|---|---|---|---| +| Overview → Intelligence | directories with `.claude-flow/neural/`, from the 150 most recent transcripts | basename | **4** | +| Observability → History | git-root identity over a 14-day transcript window | identity hash | **14** | +| Usage → Scorecard | `projectLabel()` basename heuristic, over its own day window | cwd basename | **14** (a *different* 14) | +| System → Projects | every `cwd` any host ever recorded | resolved real path | **~48** | + +No single number was wrong. What was wrong is that nothing told the user which question each was +answering, so four honest answers read as one broken feature. Worse, the Intelligence number was +answering a question nobody asked: `.claude-flow/neural/` means "ruflo has *trained* here", which +silently excluded every project whose memory came from agentic-qe or swarm storage, and every +project driven by Codex or OpenCode rather than Claude. On the deciding machine it reported 4 +projects where 17 had memory or intelligence active. + +## Decision + +**One census. Four named scopes. Every count is rendered with the scope that produced it.** + +`discoverProjectSources()` (`src/lib/footprint/project-sources.mjs`) becomes the single census, +reused verbatim rather than reimplemented. It is already the widest and most carefully bounded of +the four sources: it reads exactly one field — the session `cwd` — out of the head of every Claude +and Codex transcript plus the OpenCode session store, dedupes by resolved real path, and reports +three deliberately distinct figures instead of one lossy total. + +`src/lib/project-census.mjs` wraps it with the scope vocabulary: + +| Scope | Means | Level | +|---|---|---| +| `everSeen` | every project any host ever recorded a session in, deletions included | directory | +| `onDisk` | the subset that still resolves — the only projects that can be measured | directory | +| `gitRepos` | the on-disk subset under version control | directory | +| `learning` | on-disk projects with `.claude-flow`, `.agentic-qe` or `.swarm` state, whichever host created it | **project** | + +Three consequences of that table are the actual decision: + +**1. The learning scope is about activation, not training.** Its markers are exactly the ones +`defaultStorageRoots()` already treats as a project's learning stores, so the Intelligence panel and +the Storage panel cannot disagree about what a learning store *is*. Host-independence is the point: +memory activated by Codex counts. + +**2. Directory scopes and project scopes are different levels, deliberately.** The System area +measures **directories** because that is what has bytes and lines in it ([ADR-0025](0025-machine-footprint-metrics.md)); +folding them would destroy the figures it exists to report. The Intelligence panel aggregates +**projects**, because a project is what a user picks. So the learning scope folds a repository's +sub-directories and its ephemeral `.claude/worktrees/agent-*` checkouts onto the repository root, +keyed by `resolveProjectIdentity()` — the same git-root identity Observability already uses. + +This is not cosmetic. The project picker keys on that identity. Listing 24 directories against 17 +identities made 7 rows unreachable and let the wrong project be selected. + +**3. Counts still differ, and that is correct.** A lifetime census, a 14-day session window and a +learning-state subset are three questions. Shared identity removes the *spurious* differences — +`backend` and `emailibrium` are no longer two projects — and leaves the real ones. For the real ones +the answer is not to force agreement but to explain: `describeScope()` returns one sentence naming +what a count counted and how it narrowed, and **no surface may render a project count without one.** +Where a count is windowed, the window is named, because that is then the only remaining reason two +counts legitimately differ. + +## Consequences + +- The Intelligence panel reports every project with memory or intelligence state, whichever host + created it — 17 rather than 4 on the deciding machine. +- The redundant "N projects tracked on this machine" and "N projects available" captions are + **deleted**, not reworded: with a scope explainer and a KPI card, they restated a number twice and + explained it zero times. +- `discoverRuvfloProjects()` and `project-discovery.mjs` are retired, and `registryWorkspaces()` + reverts to module-private. ruflo's machine-level registries do not exist on every machine — which + is exactly why they made a poor discovery source and a fine daemon-accounting one. +- A census walk is not free (~650 ms over ~3,200 transcripts on the deciding machine). It stays + behind the existing 60-second snapshot cache, walked once per snapshot and exposed two ways. +- `learningState` rides onto each project row so a project reporting zero patterns can say *why* — + a project with `.agentic-qe` but no ruflo counters is genuinely active and genuinely has no + patterns, and without that field it is indistinguishable from a failed read ([ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md)). +- The census reports `complete: false` when a transcript could not be read, and every figure derived + from it is then rendered as a lower bound rather than a fact. +- Adding a fifth scope means adding a row to `SCOPE_NOTE`; a scope with no explanation returns the + empty string rather than rendering an unexplained number. + +## References + +- `src/lib/project-census.mjs` — `projectCensus`, `projectsInScope`, `describeScope`, + `hasLearningState`, `LEARNING_MARKERS` +- `src/lib/footprint/project-sources.mjs` — `discoverProjectSources` (the census itself) +- `src/lib/live/project-label.mjs` — `resolveProjectIdentity` (the shared identity) +- `src/lib/dashboard-server.mjs` — `censusBackedDiscovery`, `buildProjectSnapshotCache` +- `tests/kit/project-census.test.mjs` +- [docs/ddd/project-intelligence.md](../ddd/project-intelligence.md), + [docs/ddd/context-map.md](../ddd/context-map.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index befc467..4e39037 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,6 +35,7 @@ Consequences**, and cites the grounded source it rests on where relevant. | [0024](0024-project-intelligence-telemetry.md) | Project intelligence: live learning telemetry from ruflo/agentic-qe's own state | Implemented | | [0025](0025-machine-footprint-metrics.md) | Machine footprint: infrastructure metrics for install, runtime, storage, and catalog | Implemented | | [0026](0026-about-component-directory.md) | About: a component directory that explains everything ak installs | Implemented | +| [0027](0027-shared-project-census.md) | One project census, four scopes, every count explains itself | Implemented | Theme: ADRs **0001–0006** define **dual-host LLM routing and leadership** — how `ak` lets ruflo route each development activity (architecture, implementation, testing, review, …) to the right host (Claude @@ -170,3 +171,19 @@ test rather than a review hope — every managed tool must have exactly one entr exist for something ak neither installs nor configures. Official marks appear only where the dashboard already ships them as official; everything else gets an explicit monogram tile rather than a fabricated logo. `ak about` prints the same directory in a terminal. + +**0027** ends a four-way disagreement about what a project *is*. Overview, Usage, Observability and +System each discovered projects their own way, from their own source, with their own naming rule, +and reported four different numbers for the same machine — 4, 14, another 14, and ~48. None was +wrong; nothing said which question each answered, so four honest answers read as one broken +feature. It makes `discoverProjectSources()` the single census and names four **scopes** over it +(`everSeen`, `onDisk`, `gitRepos`, `learning`), with the rule that no surface may render a project +count without the sentence explaining what that count counted. Two decisions carry the weight. The +*learning* scope asks whether memory or intelligence has been **activated** — `.claude-flow`, +`.agentic-qe` or `.swarm`, whichever host created it — rather than whether ruflo has *trained*, +which is why Intelligence went from 4 projects to 17 on the deciding machine. And directory scopes +stay separate from project scopes: System measures directories because that is what has bytes in +it, while Intelligence folds a repo's sub-directories and throwaway agent worktrees onto one +identity because that is what a user picks — a distinction that was also a live bug, since keying +the picker off identity while listing directories made 7 of 24 rows unreachable. Counts that remain +different stay different, and say why. diff --git a/docs/ddd/context-map.md b/docs/ddd/context-map.md index ddc0777..a57816a 100644 --- a/docs/ddd/context-map.md +++ b/docs/ddd/context-map.md @@ -146,12 +146,29 @@ and credential policy is distinct from the offline-first dashboard and integrati | Project state (`.claude-flow/*`) | Project intelligence | Direct local file reads; no anti-corruption adapter needed | | Project intelligence | Dashboard delivery | Read-model projection, delivered by poll (`/api/status`) and SSE push (`/api/live/intelligence`) | | Local filesystem and process table | Machine footprint | Direct metadata-only reads; no anti-corruption adapter needed | -| Project discovery | Machine footprint | Candidate paths only; every rendered figure is measured by this context's own collectors | +| Project census | Machine footprint | Candidate paths only, at directory granularity; every rendered figure is measured by this context's own collectors | +| Project census | Project intelligence | The `learning` scope, folded onto project identity — the project list and the selectable key | +| Project census | Historical usage | Repository roots, so a session in a sub-directory labels as its repository rather than as a peer project | | Machine footprint | Dashboard delivery | Two-tier measurement read model over `GET /api/system`, and the same collector behind `ak system` | | Integration management | Component directory | Registry consumed only as a parity gate; no editorial content flows either way | | Detection facts (`/api/status`, managed-tools) | Component directory | Read-only join at render; a failed join degrades chips to unknown, never hides cards | | Component directory | Dashboard delivery | Versioned editorial entries imported by the page; no endpoint, no probe, no cache | +### Project census (shared kernel) + +`src/lib/project-census.mjs` is the one enumeration of this machine's projects +([ADR-0027](../adr/0027-shared-project-census.md)). It is a **shared kernel**, not a context: it +owns no domain logic, produces no rendered figure, and every consumer applies its own named scope +and takes its own measurements. Four contexts derive their project list from it, and the identity +it keys on (`resolveProjectIdentity`) is Observability's, reused rather than reinvented — which is +what makes a project mean the same thing in all four. + +The kernel deliberately serves **two granularities**. Machine footprint consumes directories, +because directories are what have bytes and lines in them. Project intelligence consumes projects, +folded onto identity, because a project is what a user selects. Collapsing those would either +destroy the System area's per-directory figures or break the Intelligence picker; both were +observed before the split was made explicit. + ## Boundary rules - Native configuration and evidence schemas never become the canonical model by accident. @@ -160,6 +177,8 @@ and credential policy is distinct from the offline-first dashboard and integrati - Dashboard presentation cannot upgrade provenance. - Historical usage and live topology share identifiers, not aggregate ownership. - Network egress occurs only in commands and contexts whose contract explicitly permits it. +- Every project count is rendered with the scope that produced it; two contexts may report + different totals, but neither may report an unexplained one. - Project intelligence reads local project state directly; it never enters Evidence Acquisition's anti-corruption layer or Observability's canonical event model, and it establishes no session, actor, host, provider, or lifecycle identity. diff --git a/docs/ddd/machine-footprint.md b/docs/ddd/machine-footprint.md index ada5132..7de1bbb 100644 --- a/docs/ddd/machine-footprint.md +++ b/docs/ddd/machine-footprint.md @@ -222,14 +222,27 @@ measured zero still says where it looked. ### Runtime census A point-in-time table of live host processes — reusing the existing current-user, argv-minimized -survey and extending its `ps` read with `pcpu`/`rss` — plus the daemon census (count, age -against the 12h TTL, budget state) and a child/MCP-server process count. The census is +survey and extending its `ps` read with `pcpu`/`rss` — plus the daemon census (count, and the +oldest daemon's age against the 12h TTL). The census is **ephemeral**: computed per request, never persisted into the snapshot file, because a process table is a moment, not a fact worth retaining, and persisting it would create a stale-liveness trap. `snapshot.mjs` enforces this structurally — it serializes only the deep-tier keys (`install`, `storage`, `catalog`, `projects`, `consumers`), so a census handed to it is dropped rather than written. +Two figures this census once carried are deliberately gone. A **launch-budget** field could never +be populated: ruflo exposes no local state to read it from, so it was a permanent "unavailable" +rather than an honest degradation, and a permanently unknowable quantity is removed rather than +rendered (see [ADR-0023](../adr/0023-fail-closed-operations-and-explicit-degradation.md) §9). A +**child/MCP-server process count** is still computed by the survey — it is what makes the per-host +rows correct — but is no longer republished: as a rendered figure it was a bare number with no +denominator, no history, and no action attached to it. + +The census is also the one deep-tier neighbour that refreshes on the dashboard's ordinary poll +clock while its view is open. It measures liveness, so a figure loaded once when the tab was first +opened is the one kind of staleness this area cannot tolerate — but only the cheap tier polls; the +filesystem walk stays behind an explicit rescan. + On **Windows** the census is real, not unsupported. `src/lib/live/win-process-survey.ps1` — a plain text script invoked the way the POSIX path already invokes `ps` and `lsof`, with no npm dependency and no compiled artifact — returns a guaranteed census (host, pid, ppid, start time, CPU, working @@ -257,6 +270,16 @@ a stated age, superseded cache snapshots, regenerable package caches, redundant revisions, extra runtime versions, orphaned worktrees), each carrying its rationale and its path. Candidates are information, not actions — this context has no delete verb. +**Learning stores are reported on their own, not mixed into the shared charts.** On a real +machine they are ~99% of retained bytes, so a donut that includes them renders as a solid ring +and a per-host bar chart flattens every other series to a sliver — the chart stops being a +measurement and becomes a picture of one category. They get a single-figure card; the donut and +the per-host split cover the remaining three categories and **state the exclusion on the panel**. +This is presentation only: the collector measures all four categories and `ak system --json` +emits all four, unchanged. The per-host split likewise shows only real hosts (claude, codex, +opencode) and names the remaining bytes — ak's own state — in a footnote with its figure, so the +bars still account for the whole of the donut beside them rather than silently disagreeing with it. + Two honest limits belong with the numbers. **Growth is approximate and says so**: a file contributes its whole size on its mtime day, which is exact for append-only transcripts and over-counts rewritten SQLite ledgers, so the figure carries its own `basis` string. And **Codex @@ -354,6 +377,17 @@ rows describe overlapping paths (an aged transcript can also sit under a project exists) reports its total as unknown-with-reason rather than counting the same bytes twice — the row count still stands, and every row still carries its own measured figure. +Reclaimables are their own area rather than a card under a measurement. Everything else in this +context reports what **is**; this is the only surface that suggests what a user might **do**, and +that difference is worth a tab of its own — buried under a byte chart it read as another statistic. +It still has no delete verb and may never gain one. + +The rows render as one scrollable table ordered regenerable-first, not as two stacked blocks of +paragraph-bearing cards. The tier pill survives that change because it is what distinguishes the +two promises at a glance, and only the regenerable pill carries bytes: on a row that may be in +use, a figure in the pill would read as "this much is yours to take back", which is a claim the +measurement does not support. A review row's bytes stay in the size column, marked as context. + The surfaces honour the split structurally: the `review` tier is rendered without a leading total at all, and its bytes ride each row as context, so the block cannot be read as "N GB available here". A row from a snapshot predating the `safety` field lands in `review` — the tier that @@ -365,11 +399,30 @@ Deduplicated `CatalogItem`s across hosts — skills, agents, commands, plugins, keyed by normalized name, each with a per-host presence matrix (which hosts carry it, from which surface it was observed). Counting is by manifest/directory-entry **names** on the host catalog surfaces Integration management already projects into; item file contents are not parsed beyond -what naming requires. The config-surface row (managed CLAUDE.md/AGENTS.md block count, settings +what naming requires. Scope is **user plus every project on disk**, not user plus the launching +repo: a skill defined in a repository is as deployed as one in `~/.claude`, and the question this +inventory answers is what the machine carries. Deduplication by `(kind, name)` means a name +defined in five projects is still one row, so the inventory grows with distinct names rather than +with project count. + +The presence matrix carries two independent multi-select filters — by kind and by host — with every +option selected at first paint, so the default remains the whole inventory. The host filter matches +**any** selected host rather than all of them: "carried by codex" is the question a reader is +asking, and intersecting would answer a different one. A filtered view states how much it is +hiding, and each row carries its own kind, because a filtered list must never leave a name +unexplained. The config-surface row (managed CLAUDE.md/AGENTS.md block count, settings file sizes) lives here because it answers the same "what is deployed" question. ### Project accounting +**The Projects table lists repositories, not directories.** A row is rendered only when it has a +remote and a host has recorded a session in it. Without that rule the table listed ephemeral +`.claude/worktrees/agent-*` checkouts, sub-folders a session happened to run in, and home +directories, each beside its own parent repository as though it were a peer with its own +multi-gigabyte figure. The session test excludes an empty host list, never a missing one — a +snapshot predating that field cannot answer the question, and reading absent as zero would blank +the table. Excluded directories are counted and characterised beneath the table. + **Two numbers, not one.** `discoverProjectSources()` publishes `everSeen` and `onDisk`, and they are different questions: diff --git a/docs/ddd/project-intelligence.md b/docs/ddd/project-intelligence.md index d0208f3..a630bc8 100644 --- a/docs/ddd/project-intelligence.md +++ b/docs/ddd/project-intelligence.md @@ -4,9 +4,14 @@ This document describes the domain implemented by [ADR-0024](../adr/0024-project `src/lib/dashboard/intel-history.mjs`, `src/lib/dashboard/project-discovery.mjs`, and `src/lib/live/intelligence-watch.mjs`. +> **2026-08-07 amendment:** project discovery moved to the shared census +> ([ADR-0027](../adr/0027-shared-project-census.md)). "Every ruflo-initialized project" below now +> reads "every project with memory or intelligence state, whichever host created it" — see +> [Project discovery](#project-discovery). +> > **2026-08-05 amendment:** extended from one project's telemetry, implicit and bound to the > dashboard server's own launching working directory, to a machine-wide catalog of every -> ruflo-initialized project on this machine, a machine-wide aggregate that is always shown, and an +> project on this machine, a machine-wide aggregate that is always shown, and an > explicitly selected, explicitly labeled detail project (defaulting to most-recently-active). See > [ADR-0024](../adr/0024-project-intelligence-telemetry.md)'s update note for the full amendment > record, including why the `/api/status` payload shape is a clean break rather than a preserved @@ -66,50 +71,42 @@ never a remote or unregistered one. ## Project discovery -`discoverRuvfloProjects()` (`src/lib/dashboard/project-discovery.mjs`) returns every project on -this machine ruflo has genuinely initialized — a `.claude-flow/neural/` subdirectory present, not -merely a bare `.claude-flow/` (a project that only ever ran, say, `ruflo daemon start` without ever -training or learning anything is correctly excluded) — deduplicated by resolved absolute path and -sorted most-recently-active first by `.claude-flow/neural/stats.json`'s `lastAdaptation`. - -Three sources are unioned: - -1. **Registry.** `registryWorkspaces()`, reused verbatim from `daemons.mjs` (imported, not - reimplemented), walks `~/.claude-flow/{ai-jobs.json,workspace-leases.json,repo-supervisors.json}` - for every workspace path recorded there that carries a `.claude-flow` directory. `daemons.mjs`'s - own header comment assumes ruflo 3.28+ reliably writes these; verified false on a real ruflo - 3.34.0 machine with real, populated ruflo projects — none of the three files existed. Kept as a - source (cheap, and correct wherever those files do exist), but no longer described as the - guaranteed-correct primary. -2. **Observability cross-reference.** `WorkspaceSnapshotStore` (`src/lib/live/workspace-store.mjs`, - [Observability](observability.md)) is checked for any record whose workspace carries a - genuinely resolvable absolute path. That store's own privacy sanitizers — `repositoryLabel` - rejects any path separator, `directoryLabel` rejects anything absolute-looking — mean a real - record never carries one, so this source is structurally empty by that store's own design, not - a gap. The check remains a real, defensive one rather than being skipped outright, so it starts - contributing automatically if that schema ever grows a genuine path field. -3. **Transcript content (the source that matters in practice).** Real absolute `cwd` values read - directly out of Claude and Codex transcript content under `~/.claude/projects/**/*.jsonl` and - `~/.codex/sessions/**/*.jsonl`, via the same `discoverJsonl()`/`bootstrapRecords()` functions - `live-sessions-service.mjs` already trusts for Observability's own live session tracking — flat - `record.cwd` for Claude, `record.payload.cwd` for Codex's `session_meta`/`turn_context` records. - Unlike source 2's sanitized, persisted registry, raw transcripts are not sanitized and do carry - a resolvable path — legitimately readable at this trust boundary since it's the same user, same - machine, same files Observability already parses. Bounded to the 150 most-recently-modified - transcripts per host so cost stays flat regardless of session count. On the real machine where - source 1 returned nothing, this source alone found all 4 real ruflo-initialized projects. - -Each discovered row is `{ path, label, source }`, where `label` reuses Observability's own -`resolveProjectLabel` for the same path (falling back to the bare directory name) so a project -reads identically wherever it is named, and `source` is `'registry'`, `'observability'`, -`'transcript'`, or `'both'` when a project was found by two or more sources (not necessarily -exactly two). +Intelligence does not discover projects. It consumes the **learning scope** of the shared project +census ([ADR-0027](../adr/0027-shared-project-census.md), `src/lib/project-census.mjs`), which is +the same census the System area measures directories from and the same identity Observability keys +sessions by. A project therefore means the same thing here as it does there. + +The learning scope is every project still on disk that carries **learning state** — a +`.claude-flow`, `.agentic-qe` or `.swarm` directory — regardless of which host created it. + +Two properties of that definition are deliberate and both were once wrong: + +- **Activation, not training.** The predicate this replaced required `.claude-flow/neural/`, which + answers "has ruflo trained here". That excluded every project whose memory came from agentic-qe + or swarm storage, and every project driven by Codex or OpenCode. On a real machine it reported 4 + projects where 17 had learning state. The markers above are exactly the ones the Storage context + already treats as a project's learning stores, so the two contexts cannot disagree about what a + learning store is. +- **One row per project identity, not per directory.** Sessions get recorded in a repository's + sub-directories and in ephemeral `.claude/worktrees/agent-*` checkouts. The census lists those as + the distinct directories they are — the System context measures bytes per directory — and the + learning scope folds them onto the repository root via `resolveProjectIdentity()`. Without the + fold, a picker keyed by identity silently loses every folded row. + +A merged row is anchored on the path that actually carries the learning state (the repository root +in the ordinary case), because that is the path `readIntelHistory()` reads. It keeps `paths`, every +directory that contributed, and `learningState`, which markers were found — so a project reporting +zero patterns can say *why* it reports zero rather than being indistinguishable from a failed read. + +`label` reuses Observability's own `resolveProjectLabel` so a project reads identically wherever it +is named, and the key a client echoes back as `?project=` is `resolveProjectIdentity(path).key`. ## Model ```text -discoverRuvfloProjects() -> ProjectRow[] { path, label, source } (every ruflo-initialized - project on this machine) +projectsInScope(census, 'learning') (every project on this machine + -> ProjectRow[] { path, label, paths, learningState, hosts } with memory or intelligence + state, any host — ADR-0027) .claude-flow/neural/patterns.json -> PatternStoreEntry[] { createdAt, type } .claude-flow/neural/stats.json -> GlobalLearningStats { patternsLearned, trajectoriesRecorded, diff --git a/docs/ddd/routing-and-orchestration.md b/docs/ddd/routing-and-orchestration.md index 3044c28..33b65b5 100644 --- a/docs/ddd/routing-and-orchestration.md +++ b/docs/ddd/routing-and-orchestration.md @@ -43,6 +43,24 @@ Route provenance is: Refresh may update diverged seeded routes. It does not overwrite user routes. +### Divergence and retirement + +Two different things can be wrong with a route's model, and they are separate concepts. + +A route is **diverged** when the defaults have moved past its seeded model. Which side is better is +activity-dependent, so divergence is a trade to weigh, never a lag to clear: it is reported with both +models' cost-per-task characteristics and is only ever resolved by an explicit refresh. + +A model is **retired** when its host has published a withdrawal notice for it. There is no trade — +the model stops answering — so a retired model is substituted for its replacement at the point every +dispatch path reads the policy, and seeded routes naming one are rewritten on the next sync. A route +that was substituted reports the id it replaced, so a surface never silently disagrees with the file +on disk. + +Retirement is the single case in which a `user` route is overridden, and only for the run: honoring +a pin into a withdrawn model fails the work rather than respecting the intent. The persisted value is +left exactly as the user wrote it. A model that is merely superseded is diverged, not retired. + ### Primary host The primary host determines which peer leads the default table and how missing-host status is diff --git a/docs/ddd/ubiquitous-language.md b/docs/ddd/ubiquitous-language.md index cc1b903..57004d2 100644 --- a/docs/ddd/ubiquitous-language.md +++ b/docs/ddd/ubiquitous-language.md @@ -81,6 +81,24 @@ scope and at machine-wide-rollup scope alike. There is no unlabeled "this projec Intelligence: the panel always shows an explicitly selected, explicitly labeled project alongside the always-visible machine-wide rollup. See [Project intelligence](project-intelligence.md). +**Project census** — the one enumeration of this machine's projects, read from the session `cwd` +recorded in every Claude and Codex transcript plus the OpenCode session store. Every area derives +its project list from it; none discovers projects independently ([ADR-0027](../adr/0027-shared-project-census.md)). + +**Scope** — the named filter an area applies to the census, and the reason two areas can report +different totals without either being wrong. `everSeen` (all, deletions included), `onDisk` (still +resolvable), `gitRepos` (under version control) and `learning` (carries learning state). A count is +never rendered without the sentence naming its scope. + +**Learning state** — a `.claude-flow`, `.agentic-qe` or `.swarm` directory in a project: memory or +intelligence has been *activated* there, by any host. Distinct from having been *trained*, which is +what ruflo pattern counters measure and what the retired `.claude-flow/neural/` predicate required. + +**Directory scope vs project scope** — `everSeen`/`onDisk`/`gitRepos` count directories, because +directories are what have bytes and lines in them. `learning` counts projects, folding a +repository's sub-directories and its ephemeral agent worktrees onto one identity, because a project +is what a user selects. + ## Machine footprint language | Term | Meaning | diff --git a/eslint.config.mjs b/eslint.config.mjs index 2805cb3..6224dda 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -13,6 +13,11 @@ export default [ '.agents/**', 'docs/archive/**', 'coverage/**', + // Gitignored tool output, same category as coverage/: tests/ui writes + // screenshots here and debugging sessions leave browser-context scratch + // scripts behind. Linting them fails `check` for anyone who has run + // `pnpm test:ui`, over globals (document, location) that are correct there. + '.ui-artifacts/**', ], }, js.configs.recommended, diff --git a/src/commands/sync.mjs b/src/commands/sync.mjs index 4983563..cbd3ec2 100644 --- a/src/commands/sync.mjs +++ b/src/commands/sync.mjs @@ -12,7 +12,7 @@ import { OPENCODE_LIFECYCLE_ADAPTER } from '../lib/opencode.mjs'; import { runLifecycle } from '../lib/adapters/lifecycle.mjs'; import { listDaemons, staleDaemons, reap } from '../lib/daemons.mjs'; import { loadKitConfig, saveKitConfig } from '../lib/config.mjs'; -import { commandHosts, applyHosts, applyProviders, hostInstallState, installHost, applyAqeRouter, seedActivityRoutesIfMultiHost, ensureCodexMcp, ensureRufloMcpInCodex, bothHostsEnabled } from '../lib/providers.mjs'; +import { commandHosts, applyHosts, applyProviders, hostInstallState, installHost, applyAqeRouter, seedActivityRoutesIfMultiHost, migrateRetiredRoutesInConfig, ensureCodexMcp, ensureRufloMcpInCodex, bothHostsEnabled } from '../lib/providers.mjs'; import { driftReport, selfDrift } from '../lib/versions.mjs'; import { RUVECTOR_PKG, managed as ruvectorManaged } from '../lib/ruvector.mjs'; import { pruneNpxStale } from '../lib/npx.mjs'; @@ -209,6 +209,20 @@ export async function run({ flags, pkgRoot }) { // eligible (e.g. aqe upgraded ≥3.13.1 since enablement), before materializing. const seed = seedActivityRoutesIfMultiHost(cfg); if (seed.seeded) { saveKitConfig(cfg); report('routing', { ok: true, changed: true, detail: `seeded ${seed.count} activities` }); } + // Retire withdrawn models from the persisted policy. Distinct from divergence + // (which stays an explicit `ak x host refresh` decision): a retired model + // stops answering, so leaving it named on disk is a scheduled failure. Only + // seeded entries are rewritten; a user pin is reported and left alone. + const retired = migrateRetiredRoutesInConfig(cfg); + if (retired.changes.length > 0) { + if (retired.changed) saveKitConfig(cfg); + for (const c of retired.changes) { + const when = c.retiresOn ? `retires ${c.retiresOn}` : 'already withdrawn'; + report('routing', c.rewritten + ? { ok: true, changed: true, detail: `${c.activity} ${c.field}: ${c.from} → ${c.to} (${when})` } + : { ok: true, changed: false, detail: `${c.activity} ${c.field} pins ${c.from} (${when}) — user pin kept; ak runs ${c.to}` }); + } + } const router = applyAqeRouter(cfg, cwd); if (router.changed || !router.ok) report('aqe router', router); const mcp = await ensureCodexMcp(cfg, cwd); diff --git a/src/lib/daemons.mjs b/src/lib/daemons.mjs index f82fd39..447d2ea 100644 --- a/src/lib/daemons.mjs +++ b/src/lib/daemons.mjs @@ -14,11 +14,15 @@ const alive = (pid) => { /** Known-workspace discovery from ruflo's machine-level registries * (~/.claude-flow/*.json record workspaces; each workspace has - * .claude-flow/daemon.pid + daemon-state.json with startedAt). Exported - * (visibility-only change, behavior unchanged) so project-discovery.mjs can - * reuse it verbatim as its guaranteed-correct primary source instead of - * reimplementing the same registry walk. */ -export function registryWorkspaces() { + * .claude-flow/daemon.pid + daemon-state.json with startedAt). + * + * Module-private again: it was exported so the (now retired) + * project-discovery.mjs could reuse it verbatim as a project-discovery + * source. Project discovery moved to the shared census (ADR-0027), which + * reads session transcripts rather than these registries — the registries do + * not exist on every machine, which is precisely why they made a poor + * discovery source. Only daemon accounting reads them now. */ +function registryWorkspaces() { const out = new Set(); const reg = path.join(home, '.claude-flow'); for (const f of ['ai-jobs.json', 'workspace-leases.json', 'repo-supervisors.json']) { diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index 1bd3db8..d29f9ba 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -58,7 +58,7 @@ import { globalRoot } from './paths.mjs'; import { drift as ruvnetBrainDrift } from './ruvnet-brain.mjs'; import { drift as ruvectorDrift, managed as ruvectorManaged } from './ruvector.mjs'; import { loadKitConfig } from './config.mjs'; -import { resolveRoutes, routingSummary, ACTIVITIES } from './routing.mjs'; +import { resolveRoutes, routingSummary, divergedRoutes, retirementOf, ACTIVITIES } from './routing.mjs'; import { renderPage } from './dashboard/page.mjs'; import { requestRejection } from './dashboard/request-security.mjs'; import { tokenMatches } from './admin-server.mjs'; @@ -68,7 +68,7 @@ import { sseChannel, reserveClientSlot, clientGone } from './dashboard/sse.mjs'; // here because readIntelHistory() already composes it (as `.healthRing`, // forwarded verbatim below); a bare `readHealthRing` import would be unused. import { readIntelHistory, readMachineWideIntel } from './dashboard/intel-history.mjs'; -import { discoverRuvfloProjects } from './dashboard/project-discovery.mjs'; +import { projectCensus, projectsInScope, describeScope } from './project-census.mjs'; import { resolveProjectIdentity, safeProjectKey } from './live/project-label.mjs'; import { TRANSCRIPT_ROOTS, @@ -208,7 +208,7 @@ function resolveSelectedProject(projects, rawParam) { * discovery reads real machine-global state (~/.claude-flow/*.json, * ~/.config/agentic-kit/observability-workspaces.json) that a test must * never depend on. */ -function buildProjectSnapshotCache(discoverProjectsFn, machineWideIntelFn) { +function buildProjectSnapshotCache(discoverProjectsFn, machineWideIntelFn, readCensusFn) { let snapshot = null; let fetchedAt = 0; return () => { @@ -217,13 +217,39 @@ function buildProjectSnapshotCache(discoverProjectsFn, machineWideIntelFn) { const projects = discoverProjectsFn().map((project) => ( { ...project, key: keyForProject(project) } )); - snapshot = { projects, machineWide: machineWideIntelFn(projects) }; + // The scope counts that let the panel say what it counted (ADR-0027). + // Null when a caller injected its own discoverProjects: that seam yields + // a project list with no census behind it, and reporting a machine-wide + // total derived from a fixture would be a fabricated number. Absent is + // the honest reading, and the renderer omits the line rather than + // printing a zero (ADR-0023). + const census = typeof readCensusFn === 'function' ? readCensusFn() : null; + snapshot = { projects, machineWide: machineWideIntelFn(projects), census }; fetchedAt = now; } return snapshot; }; } +/** The default discovery seam: one census walk, exposed two ways — the + * learning-scoped project rows the Intelligence panel aggregates, and the + * scope counts that explain how that number relates to what the other tabs + * show. Paired so the corpus is walked ONCE per snapshot rather than twice. */ +function censusBackedDiscovery() { + let last = null; + return { + discover: () => { + last = projectCensus(); + return projectsInScope(last, 'learning'); + }, + readCensus: () => (last ? { + everSeen: last.everSeen, onDisk: last.onDisk, + gitRepos: last.gitRepos, learning: last.learning, + complete: last.complete, + } : null), + }; +} + /** Assemble the full /api/status payload. */ async function collectData({ cwd, fetchStatus, projectParam, getProjectSnapshot }) { let status; @@ -261,7 +287,7 @@ async function collectData({ cwd, fetchStatus, projectParam, getProjectSnapshot // `getProjectSnapshot()` is the shared, TTL-cached {projects, machineWide} // read; `projectParam` is the raw ?project= query value, resolved the exact // same way /api/live/intelligence resolves it (resolveSelectedProject). - const { projects, machineWide } = getProjectSnapshot(); + const { projects, machineWide, census } = getProjectSnapshot(); const selected = resolveSelectedProject(projects, projectParam); const selectedHistory = selected ? readIntelHistory(selected.path) : EMPTY_SELECTED_HISTORY; @@ -316,6 +342,18 @@ async function collectData({ cwd, fetchStatus, projectParam, getProjectSnapshot patternStore: selectedHistory.patternStore, graph: selectedHistory.graph, machineWide, + // census — how this panel's project count relates to the counts the + // other tabs show (ADR-0027). `scope` names the filter Intelligence + // applies; `counts` are the same machine census under every scope, so + // a user can see that 14 and 48 and this number are three questions + // rather than three answers. Null when discovery was injected — there + // is no census behind a fixture, and inventing one would be worse + // than saying nothing. + census: census ? { + scope: 'learning', + note: describeScope('learning'), + counts: census, + } : null, }, routing: routingPayload(), }; @@ -328,15 +366,28 @@ export function routingPayload(cfg = loadKitConfig()) { const policy = cfg.routing?.routes ?? {}; if (!Object.keys(policy).length) return null; const routes = resolveRoutes(policy); + // Two DIFFERENT signals, deliberately kept apart on the wire so the panel can + // say different things about them (see RETIRED_MODELS in routing.mjs): + // retiredFrom — the host withdrew this model; ak already substituted it, so + // the row shows what will actually run. Actionable but not a + // choice. + // diverged — a seeded route the defaults moved past. A trade to weigh, + // cleared only by an explicit `ak x host refresh`. + const diverged = new Map(divergedRoutes(policy).map((d) => [d.activity, d])); return { primaryHost: cfg.routing?.primaryHost ?? 'claude', summary: routingSummary(policy), routes: ACTIVITIES.map((activity) => { const r = routes[activity]; + const d = diverged.get(activity); return { activity, host: r.host, model: r.model ?? '', provenance: r.provenance, akOriginated: !!r.akOriginated, escalation: (r.escalation ?? []).map((e) => e.host), + ...(r.retiredFrom + ? { retiredFrom: r.retiredFrom, retiresOn: retirementOf(r.retiredFrom)?.retiresOn ?? null } + : {}), + ...(d ? { diverged: { defaultModel: d.defaultModel, defaultNote: d.defaultNote, currentNote: d.currentNote } } : {}), }; }), }; @@ -731,9 +782,12 @@ export function startDashboard({ // server, used by BOTH /api/status and /api/live/intelligence so they can // never scan the machine independently or disagree on results within the // same TTL window. + const censusBacked = censusBackedDiscovery(); + const injectedDiscovery = typeof discoverProjects === 'function'; const getProjectSnapshot = buildProjectSnapshotCache( - typeof discoverProjects === 'function' ? discoverProjects : discoverRuvfloProjects, + injectedDiscovery ? discoverProjects : censusBacked.discover, typeof machineWideIntel === 'function' ? machineWideIntel : readMachineWideIntel, + injectedDiscovery ? null : censusBacked.readCensus, ); // ── /api/live/intelligence pool plumbing ────────────────────────────────── diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index 157e35f..ef4f86c 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -100,7 +100,7 @@ export const JS = ` var AREAS={about:"panel-about",overview:"area-overview",usage:"panel-usage", observability:"panel-observability",system:"area-system"}; var OVERVIEW_VIEWS=["summary","hosts","providers","runtime","intel"]; - var SYSTEM_VIEWS=["summary","storage","runtime","catalog","projects"]; + var SYSTEM_VIEWS=["summary","advisory","sessions","storage","runtime","catalog","projects"]; var ABOUT_SECTIONS=["hosts","engine","quality","kit","configured"]; var VIEWS=["score","limits","findings","sessions","transcript"]; var CAT=${CAT_JS}; @@ -462,8 +462,20 @@ export const JS = ` var curveArr=(imp&&Array.isArray(imp.curve))?imp.curve:[]; var curveVals=curveArr.map(function(c){return Number(c&&c.acc)||0;}); - if(!pats.length&&!deltas.length&&!storeSeries.length&&!nodesSeries.length&&!curveVals.length){strip.hidden=true;return;} + // The project PICKER lives in this strip's head, so the strip itself must + // stay visible even when the selected project has nothing to chart — + // hiding it would strand the user on an empty project with no control to + // pick a different one. Only the charts collapse. + var sparkRow=document.getElementById("spark-row"); + var emptyEl=document.getElementById("history-empty"); + var nothing=!pats.length&&!deltas.length&&!storeSeries.length&&!nodesSeries.length&&!curveVals.length; strip.hidden=false; + if(sparkRow)sparkRow.hidden=nothing; + if(emptyEl){ + emptyEl.hidden=!nothing; + if(nothing)emptyEl.textContent="no learning history recorded for "+(selectedProjectLabel||"this project")+" yet."; + } + if(nothing){note.textContent="";return;} note.textContent=(series.length?series.length+" samples":"snapshot")+(intelSource?" · live":""); document.getElementById("spark-patterns").innerHTML=pats.length>1?sparkline(pats):flat(pats.length?String(pats[0])+" (one sample)":"no data"); @@ -513,17 +525,43 @@ export const JS = ` }; } + // The census explainer (ADR-0027). Two counts on two tabs are allowed to + // differ — they answer different questions — but the user must be able to + // find out WHICH question each answered. This is that affordance; it is the + // reason the redundant "N projects tracked on this machine" caption could be + // dropped rather than merely reworded. + function renderCensus(census){ + var box=document.getElementById("mw-census"); + var body=document.getElementById("mw-census-body"); + if(!box||!body)return; + // No census means discovery was injected — say nothing rather than invent a + // machine-wide figure from a fixture. + if(!census||!census.counts){box.hidden=true;body.innerHTML="";box.open=false;return;} + var c=census.counts; + var html='

This panel counts '+fmtNum(c.learning)+' project' + +(c.learning===1?"":"s")+" — "+esc(census.note||"")+".

"; + html+='

Measured from '+fmtNum(c.everSeen)+' project' + +(c.everSeen===1?"":"s")+" any host has ever recorded a session in. Of those, " + +fmtNum(c.onDisk)+" still exist on disk and "+fmtNum(c.gitRepos) + +" are git repositories. Other tabs count over their own time window, " + +"so a smaller number there is a shorter window, not a missing project.

"; + html+='

Directories that belong to one repository — a sub-folder a ' + +"session ran in, or a throwaway agent worktree — are folded into that one project here.

"; + if(c.complete===false){ + html+='

At least one transcript could not be read, ' + +"so every figure above is a lower bound.

"; + } + body.innerHTML=html; + box.hidden=false; + } + function renderMachineWide(mw){ var totals=(mw&&mw.totals)||{}; var perProject=Array.isArray(mw&&mw.perProject)?mw.perProject.slice():[]; - var note=document.getElementById("mw-note"); - if(note)note.textContent=totals.projectCount - ?(fmtNum(totals.projectCount)+" project"+(totals.projectCount===1?"":"s")+" tracked on this machine") - :"no ruflo-initialized projects discovered on this machine"; var hero=document.getElementById("mw-hero"); if(hero)hero.innerHTML= kpi("patterns learned",fmtNum(totals.patternsLearnedLifetime),"lifetime · every tracked project","") - +kpi("projects tracked",fmtNum(totals.projectCount),"ruflo-initialized on this machine","") + +kpi("projects tracked",fmtNum(totals.projectCount),"with memory or intelligence state","") +kpi("most active project",totals.mostActiveProject||"—","by most recent learning adaptation","accent"); perProject.sort(function(a,b){return (Number(b&&b.patternsLearned)||0)-(Number(a&&a.patternsLearned)||0);}); var table=document.getElementById("mw-table"); @@ -535,8 +573,19 @@ export const JS = ` var p=perProject[i]||{}; var lastMs=Number(p.lastAdaptation)||0; var lastTxt=lastMs?ago(Math.max(0,Math.round((Date.now()-lastMs)/1000))):"—"; + // Name the stores this project actually has. Without it a 0/0/— row is + // ambiguous between "intelligence is active here but ruflo never trained" + // and "something failed to read" — and the first is the common case now + // that the panel counts agentic-qe and swarm state too. + var stores=Array.isArray(p.learningState)?p.learningState:[]; + var storeTip=stores.length?stores.join(" · "):"no learning stores found"; + var storeHtml=stores.length + ? '' + +stores.map(function(s){return '';}).join("") + +"" + : ""; html+='
' - +''+esc(p.label||"(unlabeled)")+"" + +''+esc(p.label||"(unlabeled)")+storeHtml+"" +''+esc(fmtNum(p.patternsLearned))+"" +''+esc(fmtNum(p.patternStoreCount))+"" +''+esc(lastTxt)+"" @@ -558,10 +607,6 @@ export const JS = ` selectedProjectKey=intel.selectedProjectKey; selectedProjectLabel=intel.selectedProjectLabel; } - var note=document.getElementById("intel-picker-note"); - if(note)note.textContent=intelProjects.length - ?(intelProjects.length+" project"+(intelProjects.length===1?"":"s")+" available") - :"no projects discovered"; var nameEl=document.getElementById("history-project-name"); if(nameEl)nameEl.textContent=selectedProjectLabel||"—"; var sel=document.getElementById("intel-project-select"); @@ -612,10 +657,30 @@ export const JS = ` var tag=r.akOriginated?' ak':''; var escHtml=(r.escalation&&r.escalation.length)?'↑ '+esc(r.escalation.join("→"))+"":""; var primAttr=(r.host===primary)?' data-primary="1"':''; + // Two different things can be worth saying about a model, and conflating + // them would be wrong in both directions (see RETIRED_MODELS in + // routing.mjs). A retirement is not a choice — the id in kit.json no + // longer answers and ak already substituted it, so the row shows what + // will RUN and flags what it replaced. A divergence IS a choice, so it is + // stated neutrally, with both models' cost-per-task notes on the tooltip. + var flag=""; + if(r.retiredFrom){ + flag='was '+esc(r.retiredFrom)+""; + }else if(r.diverged){ + var dNote=r.diverged.defaultNote?(" — default: "+r.diverged.defaultNote):""; + var cNote=r.diverged.currentNote?(" | current: "+r.diverged.currentNote):""; + flag='default: '+esc(r.diverged.defaultModel)+""; + } html+='
' +''+esc(r.activity)+tag+"" +''+esc(r.host)+"" - +''+esc(r.model)+"" + +''+esc(r.model)+flag+"" +''+escHtml+''+esc(r.provenance)+"" +"
"; } @@ -706,6 +771,7 @@ export const JS = ` renderAbout(data); renderPanels(data.rows); renderMachineWide(data.intel&&data.intel.machineWide); + renderCensus(data.intel&&data.intel.census); renderProjectPicker(data.intel||{}); renderHistory(buildHistoryView(data)); renderRouting(data.routing); @@ -756,6 +822,60 @@ export const JS = ` try{localStorage.setItem(LS_POLL,JSON.stringify({on:pollOn,intervalMs:pollMs}));}catch(e){} } + // ── Panel collapse (.strip-toggle) ────────────────────────────────────────── + // Persisted, unlike the ephemeral row expanders: these panels are re-rendered + // by every poll, so an unpersisted collapse would reopen itself within 30 s. + // Only DEPARTURES from each panel's markup default are stored, so the default + // stays the single source of truth (provider account analytics ships + // aria-expanded="false"; the routing panels ship "true") and changing one in + // page.mjs does not need a matching change here or a localStorage migration. + var LS_COLLAPSE="ak-dash-collapse"; + var collapseState={}; + try{ + var savedCollapse=JSON.parse(localStorage.getItem(LS_COLLAPSE)||"null"); + if(savedCollapse&&typeof savedCollapse==="object")collapseState=savedCollapse; + }catch(e){} + + function saveCollapse(){ + try{localStorage.setItem(LS_COLLAPSE,JSON.stringify(collapseState));}catch(e){} + } + + function setStripCollapsed(btn,collapsed){ + var id=btn.getAttribute("aria-controls"); + var body=id?document.getElementById(id):null; + btn.setAttribute("aria-expanded",collapsed?"false":"true"); + if(body)body.hidden=collapsed; + if(id){collapseState[id]=collapsed;saveCollapse();} + } + + // Collapsible panels live in more than one tab (Overview's routing pair, the + // Usage scorecard's provider analytics), so the listener is DOCUMENT-level + // rather than hung off any one panel — the per-panel listeners each only fire + // inside their own subtree, which would silently leave the others inert. + // Then apply saved state once: panels whose id was never toggled keep + // whatever page.mjs declared. + function wireStripCollapse(){ + document.addEventListener("click",function(e){ + var btn=e.target&&e.target.closest?e.target.closest(".strip-toggle"):null; + if(!btn)return; + setStripCollapsed(btn,btn.getAttribute("aria-expanded")==="true"); + }); + // Storage's session rows open the same transcript view Usage does, through + // the public bridge rather than a second navigation path. It validates the + // id itself and returns false on a bad one, so a stale row cannot strand + // the user on an empty view. + document.addEventListener("click",function(e){ + var link=e.target&&e.target.closest?e.target.closest("[data-transcript]"):null; + if(!link)return; + window.AKDashboardOpenTranscript(link.getAttribute("data-transcript")); + }); + var btns=document.querySelectorAll(".strip-toggle"); + for(var i=0;i'+list[i]+""; return '
'+esc(label)+"" +'
'+valueHtml+"
" - +'
'+sub+"
"; + +'
'+html+"
"; } function odBytes(m){ if(!m||m.status==="unknown"||m.value==null)return mhtml(m,fmtBytes); @@ -2001,20 +2150,42 @@ export const JS = ` +''+esc(p.u+" retained")+"" +""; } - function svgArea(values,color,label){ + // A sparkline with SCALED AXES. The bare version drew a shape with no + // magnitude on it, so five of them side by side looked comparable when one + // could be a thousand times the others — the eye reads the silhouette, and + // every silhouette is normalised to its own max. The axes option adds the y peak and + // the x endpoints, which is the minimum that makes two panels comparable. + // Gutters are reserved in the viewBox rather than overlaid, so a long byte + // label can never sit on top of the plot. + function svgArea(values,color,label,axes){ if(!values.length)return sysEmpty("no days measured"); - var W=180,H=54,max=Math.max.apply(null,values)||1,pts=[],i; + var PAD_L=axes?34:0,PAD_B=axes?12:1,W=180,H=54; + var plotW=W-PAD_L,plotH=H-PAD_B-4; + var max=Math.max.apply(null,values)||1,pts=[],i; for(i=0;i' - +'' - +'' + var line=pts.join(" "),last=pts[pts.length-1].split(","),base=(4+plotH).toFixed(1); + var axisHtml=""; + if(axes){ + var xFirst=(axes.firstDay||""),xLast=(axes.lastDay||""); + axisHtml= + // y axis: peak and zero. Two ticks, not a scale — a five-tick axis in a + // 54px band is unreadable and the peak is the number that matters. + ''+esc(axes.peakLabel||"")+"" + +'0' + +'' + +''+esc(xFirst)+"" + +''+esc(xLast)+""; + } + return '' + +'' + +'' +'' +'' + +axisHtml +""+esc(label)+"" +""; } @@ -2099,18 +2270,20 @@ export const JS = ` // EXCLUDED from the sum and named in the caption, so the total is an honest // floor over the projects that were counted rather than a figure that // silently treats an unreadable project as containing no code. + /** The projects tile's sub-lines, one fact per entry — led by the on-disk + * count, which the headline no longer carries. */ function locSummary(projects){ - if(!projects)return unkHtml("projects have not been deep-scanned yet",true)+" lines"; + var onDisk=projects&&projects.onDisk?[mhtml(projects.onDisk)+" on disk"]:[]; + if(!projects)return [unkHtml("projects have not been deep-scanned yet",true)+" lines"]; var list=projects.projects||[],total=0,counted=0,i; for(i=0;imeasured'; - var head=odCount(p.everSeen)+'ever'; - if(!p.onDisk)return head; - return head+'\\u00b7 '+mhtml(p.onDisk)+" on disk"; + return odCount(p.everSeen)+'ever'; } // Liner note (data, never design): the two counts come from different // populations and the de-duplication across hosts is the part a reader cannot @@ -2170,22 +2345,27 @@ export const JS = ` var rt=d.runtime||{},totals=rt.totals||{}; var kpis=document.getElementById("sys-kpis"); if(kpis){ - // Sub lines are kept to ONE line of facts each. A KPI band is read by - // scanning down the values; a tile whose caption wraps to a second line - // pushes every sibling tile taller for prose nobody scans. + // One fact per line. A KPI band is read by scanning down the values, and a + // caption that wraps mid-separator is harder to scan than three short + // stacked lines — the tiles are a grid, so they share a height regardless. var counts=(catalog&&catalog.counts)||{}; kpis.innerHTML= - kpiCard("install footprint",odBytes(install&&install.totals&&install.totals.installBytes), - mhtml(install&&install.totals&&install.totals.toolsPresent)+" tools \\u00b7 " - +mhtml(install&&install.totals&&install.totals.nativeAddons)+" addons") + kpiCard("install footprint",odBytes(install&&install.totals&&install.totals.installBytes),[ + mhtml(install&&install.totals&&install.totals.toolsPresent)+" tools", + mhtml(install&&install.totals&&install.totals.nativeAddons)+" native addons", + ]) +kpiCard("data retained",odBytes(storage&&storage.totals&&storage.totals.bytes), - "transcripts \\u00b7 ledgers \\u00b7 caches") - +kpiCard("live processes",odCount(totals.processCount), - mhtml(totals.rssBytes,fmtBytes)+" RSS \\u00b7 " - +mhtml(totals.cpuPercent,function(v){return v.toFixed(1)+"%";})+" CPU") + ["transcripts","ledgers","caches"]) + +kpiCard("live processes",odCount(totals.processCount),[ + mhtml(totals.rssBytes,fmtBytes)+" RSS", + mhtml(totals.cpuPercent,function(v){return v.toFixed(1)+"%";})+" CPU", + ]) +kpiCard("projects",projectsValue(projects),locSummary(projects)) - +kpiCard("catalog",odCount(counts.skill), - "skills \\u00b7 "+mhtml(counts.agent)+" agents \\u00b7 "+mhtml(counts.command)+" cmds"); + +kpiCard("catalog",odCount(counts.skill),[ + "skills", + mhtml(counts.agent)+" agents", + mhtml(counts.command)+" commands", + ]); mountOdometers(kpis); } var kpiNote=document.getElementById("sys-kpis-note"); @@ -2357,52 +2537,32 @@ export const JS = ` // yours to take back", and on a row that may be in use it would be a claim // the measurement does not support. var TIER_ORDER=["regenerable","review"]; - var TIER_FALLBACK_MEANING={ - regenerable:"The owning tool refetches this on demand.", - review:"Plausible but not safe to call removable \\u2014 review each one; this is not a " - +"total to sweep." - }; + // One scrollable table rather than two stacked card grids. The rows are an + // advisory list of "here is a thing, here is where it lives" — nine of them + // as paragraph-bearing cards towered over the rest of the tab, and the + // reading task (scan, find a path, go run something elsewhere) is a table's. + // The pills stay: they are the visual grammar that separates the two + // promises, and only the regenerable one carries bytes. function reclaimRow(r){ var review=r.safety!=="regenerable"; var meas=mhtml(r.bytes,fmtBytes); - var keeps=r.keeps||[],names=[],i; - for(i=0;i' - +(review?'review' - :''+meas+"") - +"
"+esc(r.label)+"
" - // On a review row the figure is CONTEXT and says which kind: 'installed' - // is what sits at that path, not a subset anyone measured as removable. - +(review?'
'+meas+" " - +esc(r.bytesMeaning==="installed"?"installed at this path":"across the matching files") - +" \\u2014 context, not a figure to reclaim
":"") - +'
'+esc(r.rationale||"")+"
" - +(names.length?'
in use and excluded from the figure: ' - +esc(names.join(", "))+"
":"") - +'
'+esc(r.path||"")+"
" - // Documentation, not an affordance: the CLI that already owns removal, - // spelled out so the reader runs it themselves somewhere else. + return '' + +""+(review?'review' + :'regenerable')+"" + +''+esc(r.label)+"" + +(r.rationale?' \\u2014 '+esc(r.rationale)+"":"") + // Documentation, not an affordance: the CLI that already owns removal. +(r.cleanupHint?'
removal lives in ' +esc(r.cleanupHint)+"
":"") - +"
"; - } - function reclaimTier(safety,tier,rows){ - if(!rows.length)return ""; - var head=''+esc(safety)+""; - // The regenerable tier states its total; the review tier deliberately does - // not lead with one. Its bytes ride each row as context instead, so the - // block cannot be read as "N GB available here". - var figure=safety==="regenerable" - ? ""+(tier?mhtml(tier.bytes,fmtBytes):unkHtml("this snapshot carries no tier total",false)) - +" across "+esc(fmtNum(rows.length))+" row"+(rows.length===1?"":"s") - : esc(fmtNum(rows.length))+" row"+(rows.length===1?"":"s") - +' no tier total \\u2014 these are pointers, not a sum'; - var html='
'+head+figure+"
" - +'
' - +esc((tier&&tier.meaning)||TIER_FALLBACK_MEANING[safety]||"")+"
" - +'
'; - for(var i=0;i
"; + +"" + +''+esc(r.path||"")+"" + // On a review row the figure is CONTEXT, never a claim of free space — + // the title says which kind so the column stays one word wide. + +'"+meas+(review?' ctx':"")+"" + +""; } function renderSysReclaim(s){ var rec=document.getElementById("sys-reclaim"); @@ -2417,36 +2577,114 @@ export const JS = ` } var summary=s.reclaimSummary||null,tiers={},i; for(i=0;i<((summary&&summary.tiers)||[]).length;i++)tiers[summary.tiers[i].safety]=summary.tiers[i]; - var html="",placed=0; + // Regenerable first, then review — the sort is the tiering now that the two + // no longer occupy separate blocks. + var ordered=[],placed=0; for(i=0;i'+esc(fmtNum(list.length-placed))+" row(s) carried no safety tier " - +"and are not shown.
":""); + ordered=ordered.concat(members); + } + var body=""; + for(i=0;iStatusWhat it isPath" + +'Size'+body+"" + +(placed'+esc(fmtNum(list.length-placed))+" row(s) carried no safety tier " + +"and are not shown.":""); if(note){ - note.innerHTML="Two tiers, never one total. " - +esc((summary&&summary.combinedNote) - ||"They are reported separately and never added: only the regenerable total is " - +"space a tool would rebuild by itself.") - +" Advisory rows only \\u2014 nothing here removes anything, by design."; - } - } + // Totals by status, still never added together — that is the one thing + // this panel's accounting must not do. + var regen=tiers.regenerable,rev=tiers.review; + var regenN=(regen&®en.rowCount)||0,revN=(rev&&rev.rowCount)||0; + note.innerHTML='regenerable ' + +(regen?mhtml(regen.bytes,fmtBytes):unkHtml("no tier total in this snapshot",false)) + +" across "+esc(fmtNum(regenN))+" row"+(regenN===1?"":"s") + +' \\u00b7 review ' + +esc(fmtNum(revN))+" row"+(revN===1?"":"s") + +' no total \\u2014 pointers, not a sum' + +'
Never added together: only the regenerable figure is space a tool ' + +"rebuilds by itself. Advisory only \\u2014 nothing here removes anything.
"; + } + } + + // Categories that dominate every other series by orders of magnitude and are + // therefore reported on their own, not mixed into a shared chart. Presentation + // only — the collector measures all four and the CLI still emits all four. + var CHART_EXCLUDED_CATEGORIES={"learning-stores":true}; + + /** The id /api/session/ answers to, derived from a storage row, or null + * when the row cannot be addressed. + * + * Storage identifies a session by its FILE BASENAME; Usage identifies it by + * a stripped id. Claude: ".jsonl" -> "". Codex: + * "rollout--.jsonl" -> "", mirroring usage-index's + * codexIdFromName. OpenCode has no transcript route at all — its whole + * store is one database file, not a session per file — so it returns null + * and the row renders unlinked rather than as a link that 404s. */ + function transcriptIdOf(row){ + if(!row||row.host==="opencode")return null; + var name=String(row.session||""); + if(name.slice(-6)===".jsonl")name=name.slice(0,-6); + if(name.indexOf("rollout-")===0){ + name=name.slice(8); + // strip a leading YYYY-MM-DDTHH-MM-SS- timestamp (19 chars + separator) + if(name.length>20&&name.charAt(10)==="T")name=name.slice(20); + } + // The same shape AKDashboardOpenTranscript validates; checking here keeps a + // malformed id from ever reaching the DOM as a control. + if(!name||name.length>128)return null; + for(var i=0;i="A"&&c<="Z")||(c>="a"&&c<="z")||(c>="0"&&c<="9")||c==="."||c==="_"||c==="-"))return null; + } + return name; + } + // The per-host split answers "which HOST is holding this". The project and + // agentic-kit keys are not hosts: project is the learning-stores pseudo-host + // (now its own card) and agentic-kit is ak's own state. Both are accounted + // for in the panel's footnote rather than dropped silently. + var REAL_HOSTS={claude:true,codex:true,opencode:true}; function renderSysStorage(d){ var s=d.storage,i,j; + + // ── learning stores, lifted out of the shared charts ── + var learn=document.getElementById("sys-learning"); + if(learn){ + if(!s){learn.innerHTML=sysEmpty(NOT_SCANNED);} + else{ + var lcat=null,lcats=s.categories||[]; + for(i=0;i" + +'
'+(lkids?"across "+esc(fmtNum(lkids))+" project store"+(lkids===1?"":"s"):"in your projects")+"
" + +'

ruflo, agentic-qe and swarm state written into your projects ' + +"(.claude-flow, .agentic-qe, .swarm). Reported on its own because it dwarfs every " + +"other category — mixed in, it flattens them to nothing.

"; + } + } + } + var donut=document.getElementById("sys-donut"); if(donut){ if(!s){donut.innerHTML=sysEmpty(NOT_SCANNED);} else{ var slices=[],legend="",total=0; for(i=0;i<(s.categories||[]).length;i++){ + if(CHART_EXCLUDED_CATEGORIES[s.categories[i].key])continue; var v=mval(s.categories[i].bytes); if(v==null)continue; slices.push({label:s.categories[i].label,value:v,color:catColor(s.categories[i].key)}); @@ -2457,8 +2695,9 @@ export const JS = ` +esc(fmtBytes(slices[i].value))+" \\u00b7 "+(total>0?pct(slices[i].value,total).toFixed(0):"0")+"%"; } donut.innerHTML=slices.length - ?('
'+svgDonut(slices,total,fmtBytes(total)+" retained by category") - +'
'+legend+"
") + ?('
'+svgDonut(slices,total,fmtBytes(total)+" excluding learning stores") + +'
'+legend+"
" + +'

Learning stores are counted on their own card, not here.

') :sysEmpty("no storage category could be measured."); } } @@ -2466,12 +2705,19 @@ export const JS = ` if(split){ if(!s){split.innerHTML=sysEmpty(NOT_SCANNED);} else{ - var byHost={},order=[],cats=s.categories||[]; + var byHost={},order=[],cats=s.categories||[],otherBytes=0,otherKeys={}; for(i=0;i'+esc(fmtBytes(row.total))+""; } var catLegend=""; - for(i=0;i'+esc(cats[i].label)+""; + for(i=0;i'+esc(cats[i].label)+""; + } + // Name what is NOT in the rows, with its figure. Dropping the non-host + // rows silently would leave the bars failing to add up to the donut + // beside them, with nothing on screen explaining the gap. + var otherNames=[]; + for(var ok in otherKeys)if(Object.prototype.hasOwnProperty.call(otherKeys,ok))otherNames.push(ok); + otherNames.sort(); + var footnote=otherNames.length + ?'

Hosts only. A further '+esc(fmtBytes(otherBytes))+" belongs to " + +esc(otherNames.join(" and "))+" — ak's own state, not a host's. Learning stores are on their own card.

" + :'

Learning stores are on their own card, not counted here.

'; split.innerHTML=order.length - ?('
'+rowsHtml+'
'+catLegend+"
") + ?('
'+rowsHtml+'
'+catLegend+"
"+footnote) :sysEmpty("no per-host storage node could be measured."); } } @@ -2503,13 +2762,22 @@ export const JS = ` if(!g||!g.hosts||!g.hosts.length){growth.innerHTML=sysEmpty(s?"no growth series was measured.":NOT_SCANNED);} else{ var panels=""; - for(i=0;i'+esc(h.host)+"" +svgArea(vals,hostColor(h.host),h.host+" \\u00b7 "+(tot==null?"total unmeasured":fmtBytes(tot)+" over "+g.windowDays+"d") - +(avg==null?"":" \\u00b7 "+fmtBytes(avg)+"/day avg")) + +(avg==null?"":" \\u00b7 "+fmtBytes(avg)+"/day avg"), + {peakLabel:fmtBytes(peak), + firstDay:dayTick(days.length?days[0].day:null), + lastDay:dayTick(days.length?days[days.length-1].day:null)}) + +'
'+(tot==null?"total unmeasured":esc(fmtBytes(tot))+" over "+esc(String(g.windowDays))+"d") + +(avg==null?"":" \\u00b7 "+esc(fmtBytes(avg))+"/day")+"
" +""; } growth.innerHTML='
'+panels+"
" @@ -2524,12 +2792,43 @@ export const JS = ` if(!s){top.innerHTML=sysEmpty(NOT_SCANNED);} else if(!sess||!sess.length){top.innerHTML=sysEmpty("no session files were measured.");} else{ - var hostTotals=storageHostTotals(s),body=""; + // Attributable rows only. A row whose project cannot be named is not a + // useful entry in a list whose whole job is "which project is holding + // these bytes" — the unattributable ones are counted in the liner + // instead, so they are excluded rather than hidden. + var attributable=[],unattributable=0; for(i=0;i0?(x.bytes/ht)*100:null; - body+=''+esc(String(x.session||"").slice(0,34))+"" + if(sess[i]&&sess[i].project)attributable.push(sess[i]);else unattributable++; + } + if(!attributable.length){ + top.innerHTML=sysEmpty("no session file could be attributed to a project."); + }else{ + var hostTotals=storageHostTotals(s),body=""; + for(i=0;i0?(x.bytes/ht)*100:null; + // Link to the transcript the same way Usage does, through the public + // bridge it already exposes. The id has to be normalised first: + // Storage's session is the FILE BASENAME, while /api/session wants + // Usage's form. A row we cannot address renders as plain text — a + // dead link is worse than no link. + var sid=transcriptIdOf(x); + // Strip the extension rather than truncating mid-id: a uuid cut at 34 + // characters reads as a corrupted value. + var sname=String(x.session||""); + if(sname.slice(-6)===".jsonl")sname=sname.slice(0,-6); + var cell=esc(sname); + body+="" + +'' + +(sid?'":cell) + +"" +''+esc(x.host||"\\u2014")+"" - +""+(x.project?esc(x.project):'not attributable')+"" + // A project whose directory is gone cannot have its name decoded + // out of the transcript directory (the encoding is lossy and is + // only reversed by walking real directories). Say that, rather than + // printing a 60-character encoded path or guessing a name from it. + +""+(x.projectResolved===false + ?'deleted project' + :esc(x.projectLabel||x.project))+"" +''+esc(fmtBytes(x.bytes))+"" +""+(share==null ?unkHtml("this host's retained total was not measured",false) @@ -2539,7 +2838,11 @@ export const JS = ` } top.innerHTML='
' +'' - +body+"
SessionHostProjectSizeShare of host
"; + +body+"" + +(unattributable?'
'+esc(fmtNum(unattributable)) + +" larger session file"+(unattributable===1?"":"s")+" could not be attributed to a project " + +"and "+(unattributable===1?"is":"are")+" not listed.
":""); + } } } } @@ -2574,12 +2877,12 @@ export const JS = ` +(rss!=null&&maxRss>0?Math.max(2,(rss/maxRss)*100):0).toFixed(1)+"%;background:"+hostColor(p.host)+'">' +''+mhtml(p.rssBytes,fmtBytes)+""; } - procs.innerHTML='
' + // pid is right-aligned in the body, so its header is too — a numeric + // column whose header hangs off the far side reads as a different column. + procs.innerHTML='
Hostpid
' + +'' +'' - +""+body+"
HostpidProjectUptimeCPURSS
" - +'
child & MCP-server processes: ' - +mhtml(rt.childProcessCount)+"" - +'observed '+esc(String(rt.observedAt||"")).slice(11,19)+" \\u00b7 never persisted
"; + +"RSS"+body+""; } } var mem=document.getElementById("sys-mem"); @@ -2601,15 +2904,22 @@ export const JS = ` var dae=document.getElementById("sys-daemons"); if(dae){ var dm=rt.daemons||{},ttl=Number(dm.ttlSecs)||0,oldest=mval(dm.oldestAgeSecs); + // Two tiles, both explained. The third — an AI-worker budget — was removed + // rather than left degraded: no code path could ever populate it, so it + // was a permanent "unavailable" teaching the reader to ignore unknowns + // (ADR-0023 SS9). + var ttlH=ttl?(ttl/3600).toFixed(0):null; dae.innerHTML='
' +'
'+mhtml(dm.count)+'
running \\u00b7 oldest ' +(oldest==null?unkHtml((dm.oldestAgeSecs&&dm.oldestAgeSecs.reason)||"no start time recorded",false) :esc((oldest/3600).toFixed(1)+"h")) - +(ttl?" of "+esc((ttl/3600).toFixed(0))+"h TTL":"")+"
" + +(ttlH?" of "+esc(ttlH)+"h TTL":"")+"
" +'
'+mhtml(dm.staleCount)+'
past TTL
' - +'
'+unkHtml((dm.budget&&dm.budget.reason)||"no readable budget state",false) - +'
AI-worker budget
' - +""; + +"" + +'

Daemons are ruflo background workers, one per active project. ' + +"Each carries a time-to-live"+(ttlH?" of "+esc(ttlH)+" hours":"") + +'; one past it has outlived its lease and is what ak x daemon-gc reaps. ' + +"Several running at once is normal \\u2014 a growing count of stale ones is not.

"; } } @@ -2657,34 +2967,126 @@ export const JS = ` countsEl.innerHTML='
'+tiles+"
"; } if(matrix){ - var items=(c.items||[]).slice(0,14),head="",body=""; - for(i=0;i'+esc(hosts[i])+""; - for(i=0;i'+esc(items[i].name)+""; - for(j=0;j=0; - body+=''; - } + // The kind heading rows are gone: they told you what you were looking at + // but gave you no way to look at less of it, and on a 318-row inventory + // the thing you want is a subset, not a signpost. Two pick lists do that + // job instead — and unlike headings they compose, so "commands carried by + // codex" is one click each rather than a scroll. + renderCatalogFilters(c); + paintCatalogMatrix(c); + } + } + + // Selected kinds/hosts, or null for "everything". Null rather than a full set + // so the default survives a payload that grows a new kind or host: an unknown + // option is INCLUDED until the user has expressed an opinion, never silently + // filtered out. + var catKinds=null,catHosts=null; + + function catalogChip(group,value,label,on){ + return '"; + } + + function renderCatalogFilters(c){ + var kinds=c.kinds||[],hosts=c.hosts||[],i,html; + var kindsEl=document.getElementById("sys-cat-kinds"); + if(kindsEl){ + html=""; + for(i=0;i=0); } - var rest=(c.items||[]).length-items.length; - matrix.innerHTML='
'+head+body+"
" - +(rest>0?'
'+esc("\\u2026"+fmtNum(rest)+" more deduplicated items measured")+"
":""); + kindsEl.innerHTML=html; + } + var hostsEl=document.getElementById("sys-cat-hosts"); + if(hostsEl){ + html=""; + for(i=0;i=0); + } + hostsEl.innerHTML=html; } } + function paintCatalogMatrix(c){ + var matrix=document.getElementById("sys-matrix"); + if(!matrix)return; + var all=c.items||[],hosts=c.hosts||[],i,j; + // The host filter asks "carried by", so a row survives if ANY selected host + // carries it — intersecting instead would answer a different question + // ("carried by all of these") and read as a bug the moment two are on. + var items=[],anyHostFilter=!!catHosts; + for(i=0;i=0)carried=true; + if(!carried)continue; + } + items.push(it); + } + var head="",body=""; + for(i=0;i'+esc(hosts[i])+""; + for(i=0;i' + +esc(items[i].name)+''+esc(KIND_LABEL[items[i].kind]||items[i].kind)+""; + for(j=0;j=0; + body+=''; + } + body+=""; + } + if(!all.length){matrix.innerHTML=sysEmpty("no catalog item was measured.");return;} + if(!items.length){ + matrix.innerHTML=sysEmpty("no item matches the current filters \\u2014 " + +fmtNum(all.length)+" measured."); + return; + } + var filtered=items.length!==all.length; + matrix.innerHTML='
' + +""+head+""+body+"
Name
" + +'
' + +(filtered + ?esc(fmtNum(items.length))+" of "+esc(fmtNum(all.length))+" deduplicated items shown" + :esc(fmtNum(all.length))+" deduplicated item"+(all.length===1?"":"s") + +" across user scope and every project on disk") + +". A dot means that host carries the name.
"; + } + + function wireCatalogFilters(){ + document.addEventListener("click",function(e){ + var t=e.target; + if(!t||!t.closest)return; + var btn=t.closest("[data-cat-kind],[data-cat-host]"); + if(!btn)return; + var isKind=btn.hasAttribute("data-cat-kind"); + var group=isKind?"kind":"host"; + var value=btn.getAttribute("data-cat-"+group); + var all=(SYSTEM&&SYSTEM.catalog&&(isKind?SYSTEM.catalog.kinds:SYSTEM.catalog.hosts))||[]; + var cur=(isKind?catKinds:catHosts)||all.slice(); + var at=cur.indexOf(value); + if(at>=0)cur=cur.slice(0,at).concat(cur.slice(at+1));else cur=cur.concat([value]); + // Turning the last one back on is "no filter", not "a filter that happens + // to match everything" — so a later payload with a new option still + // includes it. + var next=cur.length===all.length?null:cur; + if(isKind)catKinds=next;else catHosts=next; + if(SYSTEM&&SYSTEM.catalog){renderCatalogFilters(SYSTEM.catalog);paintCatalogMatrix(SYSTEM.catalog);} + }); + } + // ── project stack ── - // Registry vocabulary, read-side. LANGUAGES carry lines and go on the bar; - // frameworks, SDKs and tools carry PRESENCE ONLY and go on chips beside the - // project — a chip has no width to misread as a quantity, which is exactly why - // they are chips and not another bar. - var STACK_KIND_LABEL={framework:"framework",sdk:"SDK",tool:"tool"}; - // Ranked steps of ONE hue, not six categories. styles.mjs caps the categorical - // series at four steps because past that they stop separating at 3:1; six - // languages of one project are an ORDER, so they render as one hue's ramp and - // are named in text underneath — identity from the label, size from the width. - var LANG_STEPS=[1,.8,.63,.49,.37,.27]; - var LANG_TOP=6; + // LANGUAGES carry lines and go on the bar. Framework/SDK/tool detection still + // runs in the collector and still ships on ak system --json; it is no longer + // RENDERED here, because presence-only chips answered neither question this + // table asks (how big, and in what) while owning half of every row. + var LANG_STEPS=[1,.8,.63,.49,.37]; + var LANG_TOP=5; /** Ranked languages, from the registry projection when present and from the * older byLanguage map when the snapshot predates it. */ function locLanguages(loc){ @@ -2726,36 +3128,46 @@ export const JS = ` } /** Frameworks / SDKs / tools as chips. Capped, with the remainder named on * hover rather than dropped. */ - function stackChips(stack){ - // A missing field means a snapshot older than the registry — not measured, - // which is a different statement from "this project uses nothing". - if(!stack)return '
stack not measured
'; - if(stack.status==="unknown") - return '
stack not measured
'; - var items=stack.items||[],chips="",i,rest=[]; - for(i=0;i'+esc(items[i].name)+""; - }else rest.push(items[i].name); - } - if(rest.length)chips+='+' - +esc(fmtNum(rest.length))+""; - // A measured project with no matches is a real "none", and it says so — - // rendering nothing would be indistinguishable from not having looked. - if(!chips)chips='none detected'; - return '
'+chips+"
"; - } function renderSysProjects(d){ var el=document.getElementById("sys-projects"); if(!el)return; var p=d.projects; if(!p){el.innerHTML=sysEmpty(NOT_SCANNED);return;} - var list=p.projects||[]; - if(!list.length){el.innerHTML=sysEmpty("no project was discovered on this machine.");return;} + var all=p.projects||[]; + if(!all.length){el.innerHTML=sysEmpty("no project was discovered on this machine.");return;} + // Repositories, not directories. Two conditions, both required: + // + // a remote — a row with no https remote is, in practice, never a project + // you would recognise: it is an ephemeral .claude/worktrees/agent-* + // checkout, a sub-directory a session happened to run in + // (myrepo/backend), or a home directory someone once launched a session + // from. Those sat beside their own parent repo as if they were peers of + // it, each with its own multi-gigabyte disk figure. + // a session — this table is about projects you have actually worked in + // with a host. Discovery is session-derived today, so this holds by + // construction; asserting it anyway keeps that true if a future + // discovery source is not. + // + // The session test excludes only an EMPTY host list, never a missing one. A + // snapshot written before rows carried a hosts field cannot answer the question, + // and reading "absent" as "no sessions" would blank the whole table for + // anyone holding one — treating unmeasured as zero, which is the one thing + // this area may not do. + // + // A genuine local-only repository is excluded too. That is the cost of the + // rule, and it is why the count is stated below rather than left implied. + var list=[],excluded=0; + for(var f=0;f0; + if(linked&&hosted)list.push(cand);else excluded++; + } + if(!list.length){ + el.innerHTML=sysEmpty("no project with a remote and a recorded session was measured \\u2014 " + +fmtNum(excluded)+" measured director"+(excluded===1?"y was":"ies were")+" excluded."); + return; + } var body="",i; for(i=0;i'; } + // The remote sub-line and the stack chips are gone: this table answers + // "how big is each project and what is it written in". A forge slug and a + // row of presence-only chips answered neither, and between them they owned + // half the row's height. The project still LINKS to its remote when it has + // an https one — the affordance was worth keeping, the metadata was not. var rem=pr.remote||null,name; - if(rem&&rem.status==="linked"&&/^https:\\/\\//.test(String(rem.webUrl||""))){ + if(rem&&rem.status==="linked"&&/^https:/.test(String(rem.webUrl||""))){ name='' - +esc(pr.label)+" ↗" - +'
'+esc((rem.host||"remote")+" \\u00b7 "+(rem.slug||""))+"
"; + +esc(pr.label)+" ↗"; }else{ - name=esc(pr.label)+'
'+esc(rem&&rem.reason?rem.reason:"local only \\u2014 no git remote")+"
"; + name=esc(pr.label); } var last=mval(pr.lastActivity); - body+=""+name+stackChips(pr.stack)+"" + body+=""+name+"" +''+mhtml(pr.loc&&pr.loc.total,function(v){return "~"+fmtTok(v);})+"" +""+langCell(pr.loc)+"" +''+mhtml(pr.totalBytes,fmtBytes)+"" - +'
'+diskBar+"
" +''+(last==null?unkHtml((pr.lastActivity&&pr.lastActivity.reason)||"no readable entry",false) :esc(ago(Math.max(0,Math.round((Date.now()-last)/1000)))))+""; } + // Legend covers only what still renders: the language ramp. The disk column + // is a single figure now, and there are no chips left to explain. el.innerHTML='
' +'lines: top '+LANG_TOP+' languages, darkest first' +'' +'' +'' - +' the rest' - +'disk: tree' - +'.git' - +'node_modules' - +'chips are frameworks, SDKs and tools \\u2014 presence only, never lines
' - // The stack column needs a floor: chips that wrap one-per-line turn every - // project into a ten-line row, so the table scrolls horizontally in its own - // wrapper rather than squeezing the name column. + +' the rest' +'
' - +'' + +'' +'' - +'' + +'' +''+body+"
Project & stack
ProjectLines ≈By languageDisktree · .git · node_modulesDiskLast active
" - // The table can only hold projects that still exist — a deleted one has no - // bytes and no lines. Saying so next to the two counts is what keeps the - // shorter table from reading as a shrinking machine. - +'
' + // Three numbers now, and the gap between the last two is a filter rather + // than a fact about the machine — so it is named. Leaving the reader to + // subtract 25 from 16 and guess is the silent exclusion ADR-0023 forbids. + +'
' +(p.everSeen ?mhtml(p.everSeen)+" projects ever seen across all hosts, " - +(p.onDisk?mhtml(p.onDisk):"some")+" still on disk and listed here" + +(p.onDisk?mhtml(p.onDisk):"some")+" still on disk" :mhtml(p.count)+" projects measured (this snapshot predates the ever-seen count)") - +". Line counts are approximate: extension-bucketed, with node_modules and vendored " - +"trees excluded. Frameworks, SDKs and tools are detected by PRESENCE and carry no " - +"line count of their own.
"; - renderSysUnrecognized(p); - } - - // "Other" as a to-do list, not a rounding bucket. Every extension the registry - // could not map and every declared dependency it could not name is listed BY - // NAME with how much of it there is — which is what makes excluding an - // unmapped extension from the line count an accounted decision rather than a - // silent loss. - function renderSysUnrecognized(p){ - var el=document.getElementById("sys-unrecognized"); - if(!el)return; - if(!p){el.innerHTML=sysEmpty(NOT_SCANNED);return;} - var u=p.unrecognized; - if(!u){ - el.innerHTML=sysEmpty("this snapshot predates the stack registry, so it carries no " - +"unrecognized tail."); - return; - } - // Ranked already; the head of each list is what a release would actually - // close, and the tail past the cap is named on hover rather than dropped. - var SHOW=24; - var ext=u.extensions||[],dep=u.dependencies||[],chips="",i,rest=[]; - for(i=0;i=SHOW){rest.push(ext[i].ext+" "+fmtNum(ext[i].files));continue;} - chips+=''+esc(ext[i].ext) - +""+esc(fmtNum(ext[i].files))+""; - } - if(rest.length)chips+='+' - +esc(fmtNum(rest.length))+""; - var deps="",depRest=[]; - for(i=0;i=SHOW){depRest.push(dep[i].name);continue;} - deps+='' - +esc(dep[i].name)+""; - } - if(depRest.length)deps+='+' - +esc(fmtNum(depRest.length))+""; - // Each total is stated, or its absence is: a capped list reports null rather - // than a smaller number presented as complete. - var counts=(u.extensionsTotal==null - ?"The extension list is capped, so the distinct count is a floor." - :esc(fmtNum(u.extensionsTotal))+" distinct unmapped extension(s).") - +" "+(u.dependenciesTotal==null - ?"The dependency list is capped too." - :esc(fmtNum(u.dependenciesTotal))+" unnamed dependenc" - +(u.dependenciesTotal===1?"y":"ies")+"."); - el.innerHTML=(chips?'
'+chips+"
":"") - +(deps?'
declared dependencies the registry does not name
' - +'
'+deps+"
":"") - +(!chips&&!deps?sysEmpty("the registry named everything it saw \\u2014 a measured nothing."):"") - +'
Counted over ' - +esc(fmtNum(u.projectsMeasured||0))+" measured project(s) by registry " - +esc(String(p.registryVersion||"?"))+". "+counts - +" Unmapped extensions are excluded from the line count \\u2014 which is exactly why " - +"they are listed by name here rather than folded into it.
"; + +", "+esc(fmtNum(list.length))+" listed here." + +(excluded + ? " Excluded "+esc(fmtNum(excluded))+" measured director"+(excluded===1?"y":"ies") + +" with no remote or no recorded session \\u2014 agent worktrees, sub-folders of a " + +"repository already listed, and repositories with no remote." + : "") + +" Line counts are approximate: extension-bucketed, with node_modules and vendored " + +"trees excluded. Disk is the whole project directory, .git and node_modules included.
"; } // Freshness is a contract, not a caption (ADR-0025 §3): every deep figure on @@ -2915,8 +3272,8 @@ export const JS = ` if(!SYSTEM)return; if(SYSTEM.error){ var ids=["sys-kpis","sys-gauge","sys-consumers","sys-donut","sys-hostsplit","sys-growth", - "sys-reclaim","sys-topsessions","sys-procs","sys-mem","sys-daemons","sys-radar", - "sys-catcounts","sys-matrix","sys-projects","sys-unrecognized"]; + "sys-learning","sys-reclaim","sys-topsessions","sys-procs","sys-mem","sys-daemons","sys-radar", + "sys-catcounts","sys-matrix","sys-projects"]; var msg=sysEmpty(SYSTEM.error+(SYSTEM.reason?" \\u2014 "+SYSTEM.reason:"")); for(var i=0;i. Only `path` and - * `label` are read here; `source` is accepted but ignored. + * Machine-wide rollup — folds readIntelHistory() across every project on this + * machine that carries learning state into one totals/perProject view. * - * Investigative note on how that array is expected to be assembled (recorded - * here because this function is the actual consumer of its output): the - * PRIMARY source is daemons.mjs's (unexported) registryWorkspaces(), which - * walks ~/.claude-flow/{ai-jobs.json,workspace-leases.json,repo-supervisors.json} - * for real absolute workspace paths. A SECONDARY source was investigated — - * Observability's WorkspaceSnapshotStore (src/lib/live/workspace-store.mjs), - * persisted at ~/.config/agentic-kit/observability-workspaces.json — and - * checked against the real file on this machine. Its records never carry an - * absolute filesystem path: `directoryLabel` is validated with - * `workspaceText(..., { pathLike: true })`, which rejects anything shaped - * like an absolute path (leading `/`, a drive letter, or a `..` segment) on - * both write AND read; `repositoryLabel` is validated with `{ leaf: true }`, - * which rejects any value containing a path separator at all; and - * `project`/`projectKey` are a sanitized basename and an irreversible - * SHA-256 hash, respectively — never a path. So by that store's own privacy - * design, the secondary source can never contribute a resolvable absolute - * path for any entry; on this machine's real observability-workspaces.json - * (multiple sessions across two projects at inspection time), zero entries - * yielded one. discoverRuvfloProjects() is expected to skip every such entry - * rather than fabricate a path from a label, so in practice it is source 1 - * (the registry scan) that supplies the projects array, today. + * `projects` is the learning scope of the shared census (ADR-0027, + * src/lib/project-census.mjs): Array<{ path: string, label: string, ... }>. + * Only `path` and `label` are read here; the census's other fields ride along + * unread. One row per project IDENTITY, not per directory — the census folds a + * repo's sub-directories and its ephemeral agent worktrees onto the repo root, + * so a project is summed once here rather than two or three times. + * + * `path` is the directory whose .claude-flow/ tree is read, and the census + * anchors a merged row on the path that actually carries the learning state + * (the repo root in the ordinary case) precisely so this read lands on it. + * + * A project with no readable history degrades to nulls/zeros rather than + * throwing, so widening the census can never make this scan fail — it can only + * add rows that report nothing. * * Like readIntelHistory()'s own patternsLearned-vs-patternStore distinction, * this rollup keeps two DIFFERENT sums distinct at machine scope: @@ -204,7 +192,7 @@ export function readIntelHistory(cwd) { * "never adapted", matching readGlobalStats' own `?? 0` default), or null * when no project has adaptation data. This is a plain on-demand scan with * no caching/TTL of its own — a later caller adds that at the server layer. - * @param {Array<{ path: string, label: string }>} projects + * @param {Array<{ path: string, label: string, learningState?: string[] }>} projects * @returns {{ * totals: { patternsLearnedLifetime: number, patternStoreEntries: number, * trajectoriesRecorded: number, projectCount: number, @@ -270,6 +258,12 @@ export function readMachineWideIntel(projects) { trajectoriesRecorded: trajectories, graphLatest, lastAdaptation, + // Which learning stores the census found, carried through unread so a row + // reading 0/0/— can say WHY: a project with .agentic-qe but no + // .claude-flow has genuinely activated intelligence and genuinely has no + // ruflo pattern counters, and without this the two are indistinguishable + // from a project where the read simply failed. + learningState: Array.isArray(entry?.learningState) ? entry.learningState : [], }); } diff --git a/src/lib/dashboard/page.mjs b/src/lib/dashboard/page.mjs index bd505b4..642cdce 100644 --- a/src/lib/dashboard/page.mjs +++ b/src/lib/dashboard/page.mjs @@ -13,6 +13,14 @@ const INTEL_CSS = ` .mw-row.mw-head{background:var(--panel-2); color:var(--ink-dim); font-size:10.5px; font-weight:600; text-transform:uppercase; letter-spacing:.06em} .mw-row:not(.mw-head):hover{background:var(--panel-2)} .mw-name{color:var(--ink); overflow:hidden; text-overflow:ellipsis; white-space:nowrap} +/* Which learning stores a project carries — three tiny dots, one per store, + so a row reporting 0 patterns still says what IS active there. Colour, not + text, because the column is already tight and the title carries the names. */ +.mw-stores{display:inline-flex; gap:3px; margin-left:7px; vertical-align:middle; cursor:help} +.mw-store{width:5px; height:5px; border-radius:50%; display:inline-block; background:var(--dim)} +.mw-store[data-store="claude-flow"]{background:var(--s1)} +.mw-store[data-store="agentic-qe"]{background:var(--s3)} +.mw-store[data-store="swarm"]{background:var(--purple)} .mw-val{color:var(--ink-2); text-align:right} .mw-row.mw-head .mw-val{color:var(--ink-dim)} @media(max-width:560px){.mw-row{grid-template-columns:1fr repeat(3,minmax(60px,1fr)); gap:6px}} @@ -25,6 +33,18 @@ const INTEL_CSS = ` } .mw-picker select:focus-visible{outline:2px solid var(--accent); outline-offset:1px} .mw-picker select:disabled{opacity:.5; cursor:not-allowed} +/* The picker moved INTO this strip's head (it is a control on "learning over + time", not a panel of its own), so the head's baseline alignment has to give + way to centre alignment or the select sits low against the heading. */ +#history .strip-head{align-items:center} +#history #strip-note{margin:0 0 14px} +/* The census explainer: how this panel's project count relates to the counts + the other tabs show. Sits under the hero because it explains the number in + it. Reuses the .i-src disclosure so it costs no vertical space until asked. */ +#mw-census{margin-top:12px} +.mw-census-line{color:var(--ink-2); font-size:12px; line-height:1.55; margin:0 0 6px} +.mw-census-line b{color:var(--ink); font-weight:600} +.mw-census-caveat{color:var(--warn); font-size:11.5px; margin:6px 0 0} `; // ───────────────────────────────────────────────────────────────────────────── @@ -34,7 +54,7 @@ const INTEL_CSS = ` // Layout: five primary areas — About, Overview, Usage, Observability, and // System — share one left-aligned secondary-navigation rail. Overview owns // Summary, Hosts & Routing, Providers, Runtime, and Intelligence; System owns -// Summary, Storage, Runtime, Catalog, and Projects. Problems never hide: +// Summary, Advisory, Sessions, Storage, Runtime, Catalog, and Projects. Problems never hide: // Overview's Summary aggregates attention and child tabs retain their scoped // badges. // @@ -151,6 +171,8 @@ export function renderPage({ name, version }) { ' - +'
' - +'' - +'' - +'' - +''+body+"
ProjectLines ≈By languageDiskLast active
" + +'
' + +projSortHeader("project","Project",false) + +projSortHeader("lines","Lines \u2248",true) + +projSortHeader("language","By language",false) + +projSortHeader("disk","Disk",true) + +projSortHeader("active","Last active",true) + +""+body+"
" // Three numbers now, and the gap between the last two is a filter rather // than a fact about the machine — so it is named. Leaving the reader to // subtract 25 from 16 and guess is the silent exclusion ADR-0023 forbids. diff --git a/src/lib/dashboard/styles.mjs b/src/lib/dashboard/styles.mjs index d8396fb..cefe084 100644 --- a/src/lib/dashboard/styles.mjs +++ b/src/lib/dashboard/styles.mjs @@ -1032,6 +1032,26 @@ a.chipf{text-decoration:none; display:inline-flex; align-items:center} @media(max-width:600px){.sy-bar{grid-template-columns:1fr 64px} .sy-bar .n{grid-column:1/-1}} /* Tables */ .sy-tblwrap{overflow-x:auto} +/* Sortable headers. The whole header is the button, so the target is the width + of the column rather than the glyph. The arrow is always present — a control + that only appears on hover is invisible to anyone who never hovers, and a + column that shifts width when you point at it is worse than no affordance. + Inactive arrows sit at low opacity; the active one takes the accent, which is + how "only one column at a time" reads without a legend. */ +.sy-sortable th{padding:0} +.sy-sort{ + display:flex; align-items:center; gap:5px; width:100%; + background:none; border:0; padding:6px 8px; cursor:pointer; + font:inherit; color:var(--ink-dim); font-weight:600; text-align:inherit; + letter-spacing:inherit; text-transform:inherit; white-space:nowrap; +} +.sy-sortable th[style*="right"] .sy-sort{justify-content:flex-end} +.sy-sort:hover{color:var(--ink-2)} +.sy-sort:focus-visible{outline:2px solid var(--accent); outline-offset:-2px; border-radius:4px} +.sy-sort .sy-arrow{font-size:9px; opacity:.28; line-height:1} +.sy-sort:hover .sy-arrow{opacity:.6} +.sy-sort.on{color:var(--accent)} +.sy-sort.on .sy-arrow{opacity:1; color:var(--accent)} /* A liner directly after a table needs real separation, and a SCROLLING table needs more of it: its last row is clipped mid-glyph by design, so a caption sitting flush underneath reads as another clipped row rather than as the diff --git a/tests/ui/dashboard-ui.mjs b/tests/ui/dashboard-ui.mjs index 9d1d79e..57d1776 100644 --- a/tests/ui/dashboard-ui.mjs +++ b/tests/ui/dashboard-ui.mjs @@ -492,6 +492,21 @@ const SYSTEM_PAYLOAD = { host: 'github.com', slug: 'example/legacy', raw: 'git@github.com:example/legacy.git', }, }, + { + // Linked and worked in, but its figures were never measured. It must + // still LIST, and must sort to the bottom in BOTH directions — an + // absent figure is not a small one. + label: 'zz-unmeasured', + loc: { total: unmeasured('the working tree could not be read'), byLanguage: {} }, + treeBytes: meas(1), gitBytes: meas(1), nodeModulesBytes: meas(0), + totalBytes: unmeasured('the working tree could not be read'), + lastActivity: unmeasured('no readable entry'), + hosts: ['claude'], + remote: { + status: 'linked', webUrl: 'https://github.com/example/unmeasured', + host: 'github.com', slug: 'example/unmeasured', raw: 'git@github.com:example/unmeasured.git', + }, + }, { // Linked, but no host ever recorded a session here. label: 'never-worked-in', @@ -1227,6 +1242,51 @@ async function main() { check('the language legend matches the ramp it describes', /top 5 languages/.test(await page.$eval('#sys-projects .sy-legend', (e) => e.innerText)), 'the legend and LANG_TOP drifted apart'); + + // ── sortable headers ── + const projCol = (n) => page.$$eval(`#sys-projects tbody tr td:nth-child(${n})`, + (els) => els.map((e) => e.innerText.split('\n')[0].trim())); + const sortState = () => page.evaluate(() => ({ + aria: [...document.querySelectorAll('#sys-projects thead th')].map((t) => t.getAttribute('aria-sort')), + active: [...document.querySelectorAll('#sys-projects .sy-sort.on')].length, + })); + + const s0 = await sortState(); + const names0 = await projCol(1); + check('the table opens sorted by project name, ascending', + s0.aria[0] === 'ascending' + && JSON.stringify(names0) === JSON.stringify([...names0].sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()))), + `aria was ${JSON.stringify(s0.aria)} and the order ${JSON.stringify(names0)}`); + check('every header is a sort control', await page.$$eval('#sys-projects thead th', + (ths) => ths.every((t) => !!t.querySelector('button[data-proj-sort]'))), + 'a header without a control is a column the user cannot order by'); + + await page.click('[data-proj-sort="project"]'); + const s1 = await sortState(); + check('clicking the active column reverses it', + s1.aria[0] === 'descending' + && JSON.stringify(await projCol(1)) === JSON.stringify([...names0].reverse()), + `aria was ${JSON.stringify(s1.aria)}`); + + await page.click('[data-proj-sort="disk"]'); + const s2 = await sortState(); + check('only one column may be the active sort at a time', + s2.active === 1 && s2.aria.filter((a) => a && a !== 'none').length === 1, + `${s2.active} active control(s), aria ${JSON.stringify(s2.aria)}`); + check('a size column opens largest-first, not ascending', + s2.aria[3] === 'descending', + 'nobody opens a size column wanting the smallest row first'); + + // The rule that matters: an unmeasured figure is not a small one. + const diskDesc = await projCol(1); + await page.click('[data-proj-sort="disk"]'); + const diskAsc = await projCol(1); + // startsWith, not equality: a linked project's cell carries a trailing ↗. + const endsWithUnmeasured = (rows) => rows[rows.length - 1].startsWith('zz-unmeasured'); + check('an unmeasured row sorts LAST in both directions', + endsWithUnmeasured(diskDesc) && endsWithUnmeasured(diskAsc), + `desc ended ${JSON.stringify(diskDesc.slice(-1))}, asc ended ${JSON.stringify(diskAsc.slice(-1))} ` + + '— ranking an absent figure presents it as a measured one'); // Hand the page back exactly as the checks below expect to find it: the real // SYSTEM_STUB served again, the System area open, and its freshness label // populated. A bare reload would leave /api/system unfetched and the From 097ecf205c90767a475f8d99fb720bd52e72040a Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Sat, 8 Aug 2026 09:51:57 -0700 Subject: [PATCH 17/19] fix(test): canonicalise fixture roots the way the collectors do, and stop asserting a POSIX-only encoding on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows CI has failed since 2387a35 with 13 failures across three footprint test files. Two distinct causes, both in the harness rather than the product. realpathSync vs realpathSync.native. The fixtures canonicalised with the JS realpath; every collector canonicalises with the native one. On POSIX these agree, so it passed everywhere else. On Windows the JS realpath leaves an 8.3 short name alone (C:\Users\RUNNER~1\...) while the native one resolves it to the long form the code under test returns (C:\Users\runneradmin\...) — the same directory in two spellings, compared against each other. Fixtures now use the native variant with the same `?? realpathSync` fallback the product uses. footprint-collectors did not canonicalise at all. A POSIX-rooted encoding asserted on Windows. Two tests build a Claude transcript-directory name with `path.sep`, which yields `-a-b-c` on POSIX and `C:-Users-...` on Windows. decodeClaudeProjectDir documents a drive prefix as undecodable and returns null, so those tests were asserting the platform, not the decoder. They are POSIX-only now, and a new test asserts the refusal contract — including the Windows shape — on every platform. Also fixes a real over-claim this surfaced. Because nothing decodes on Windows, System > Sessions would have labelled EVERY row "deleted project" there. labelSessions now reports which reason applies — `gone` for a POSIX-rooted name that no longer resolves, `encoding` for a name that was never decodable — and the panel says "name not decodable" rather than asserting a deletion that did not happen. --- src/lib/dashboard/client.mjs | 13 +++++++---- src/lib/footprint/storage.mjs | 13 ++++++++++- tests/kit/footprint-collectors.test.mjs | 21 +++++++++++++++-- tests/kit/footprint-projects.test.mjs | 31 ++++++++++++++++++++++--- tests/kit/footprint-stack.test.mjs | 8 ++++++- 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index 3b51832..bb09e53 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -2839,12 +2839,15 @@ export const JS = ` +(sid?'":cell) +"" +''+esc(x.host||"\\u2014")+"" - // A project whose directory is gone cannot have its name decoded - // out of the transcript directory (the encoding is lossy and is - // only reversed by walking real directories). Say that, rather than - // printing a 60-character encoded path or guessing a name from it. + // An undecoded name says WHICH reason. "deleted project" is a + // claim, and on Windows it would be a false one for every row: the + // encoding there carries a drive prefix that the decoder refuses by + // design, so nothing is decodable and nothing has been deleted. +""+(x.projectResolved===false - ?'deleted project' + ?''+(x.projectReason==="encoding"?"name not decodable":"deleted project")+"" :esc(x.projectLabel||x.project))+"" +''+esc(fmtBytes(x.bytes))+"" +""+(share==null diff --git a/src/lib/footprint/storage.mjs b/src/lib/footprint/storage.mjs index 9c51217..f875f4f 100644 --- a/src/lib/footprint/storage.mjs +++ b/src/lib/footprint/storage.mjs @@ -570,9 +570,20 @@ export function labelSessions(rows, { decodeDir = decodeClaudeProjectDir, fsImpl // the project directory is gone — the encoding is only reversible by // walking real directories — and a consumer that cannot tell a decoded // name from an undecodable one would present a guess as a fact. + // Two different reasons a decode fails, and they mean opposite things to a + // reader. `gone`: the name is a POSIX-rooted encoding but no such + // directory exists any more — the project really was deleted. `encoding`: + // the name was never a POSIX-rooted encoding at all, which on Windows is + // every name (a drive prefix, which decodeClaudeProjectDir refuses by + // design). Collapsing the two would report every Windows session as a + // deleted project. cache.set(key, decoded ? { projectLabel: path.basename(decoded), projectResolved: true } - : { projectLabel: key, projectResolved: false }); + : { + projectLabel: key, + projectResolved: false, + projectReason: String(key).startsWith('-') ? 'gone' : 'encoding', + }); } return cache.get(key); }; diff --git a/tests/kit/footprint-collectors.test.mjs b/tests/kit/footprint-collectors.test.mjs index 77c1fa4..9076658 100644 --- a/tests/kit/footprint-collectors.test.mjs +++ b/tests/kit/footprint-collectors.test.mjs @@ -29,9 +29,16 @@ import { collectCatalog, tomlTableNames } from '../../src/lib/footprint/catalog. const DAY = 86_400_000; -/** A fixture root that is removed when the test ends, whatever the outcome. */ +/** A fixture root that is removed when the test ends, whatever the outcome. + * + * Canonicalised with realpathSync.NATIVE, matching what the collectors resolve + * paths with. The JS realpath leaves a Windows 8.3 short name alone + * (C:\Users\RUNNER~1\...) while the native one resolves it to the long form the + * code under test produces (C:\Users\runneradmin\...) — the same directory in + * two spellings, which made every path assertion here fail on Windows only. */ function fixture(t, name) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), `ak-footprint-${name}-`)); + const real = fs.realpathSync.native ?? fs.realpathSync; + const dir = real(fs.mkdtempSync(path.join(os.tmpdir(), `ak-footprint-${name}-`))); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); return dir; } @@ -1012,6 +1019,7 @@ test('an undecodable project is FLAGGED, never guessed at', () => { // decodeClaudeProjectDir returns null for a directory that no longer exists. const encoded = '-repos-gone-for-ever'; const [row] = labelSessions([{ session: 'a.jsonl', project: encoded }], { decodeDir: () => null }); + assert.equal(row.projectReason, 'gone', 'a POSIX-rooted name that will not resolve IS a deleted project'); assert.equal(row.projectResolved, false, 'a deleted project cannot be decoded and must say so'); assert.equal(row.projectLabel, encoded, 'falling back to the encoded name beats inventing one'); }); @@ -1052,3 +1060,12 @@ test('collectStorage labels the sessions it returns', (t) => { assert.equal(s.projectResolved, true); } }); + +test('an undecodable name says WHICH reason — deleted, or never encodable', () => { + // On Windows every transcript directory carries a drive prefix, which the + // decoder refuses by design. Reporting those as "deleted" would be a false + // claim about every row on that platform. + const [win] = labelSessions([{ session: 'a', project: 'C:-Users-me-proj' }], { decodeDir: () => null }); + assert.equal(win.projectResolved, false); + assert.equal(win.projectReason, 'encoding', 'nothing was deleted — the name was never decodable'); +}); diff --git a/tests/kit/footprint-projects.test.mjs b/tests/kit/footprint-projects.test.mjs index 56fd694..9d833d6 100644 --- a/tests/kit/footprint-projects.test.mjs +++ b/tests/kit/footprint-projects.test.mjs @@ -28,8 +28,20 @@ import { collectProjects } from '../../src/lib/footprint/projects.mjs'; * Realpath'd up front: macOS /tmp is a symlink to /private/tmp, and the * de-duplication under test resolves real paths, so a raw mkdtemp path would * make every assertion compare two spellings of the same directory. */ +// The encoding this module decodes is `/`-rooted — every separator became `-`, +// so the name begins with one. Windows paths carry a drive prefix instead, which +// the decoder refuses by design, so a test that builds a `/`-rooted encoding is +// only meaningful on POSIX. +const POSIX_ONLY = process.platform === 'win32'; + function fixture(t, name) { - const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-projects-${name}-`))); + // realpathSync.NATIVE, matching what the collectors canonicalise with. The JS + // realpath leaves a Windows 8.3 short name alone (C:\Users\RUNNER~1\...) while the + // native one resolves it to the long form the code under test produces + // (C:\Users\runneradmin\...). Same directory, two spellings — and every path + // assertion in this file compared one against the other on Windows only. + const real = fs.realpathSync.native ?? fs.realpathSync; + const dir = real(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-projects-${name}-`))); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); return dir; } @@ -71,7 +83,7 @@ test('a cwd is read from either host shape, and only from the head', () => { assert.equal(firstCwd([JSON.stringify({ cwd: '' })], 'claude'), null); }); -test('the encoded Claude directory decodes only against the real filesystem', (t) => { +test('the encoded Claude directory decodes only against the real filesystem', { skip: POSIX_ONLY }, (t) => { const root = fixture(t, 'decode'); // `agentic-kit` and `agentic/kit` encode IDENTICALLY, so the decode is only // safe because the filesystem decides which one exists. @@ -87,8 +99,18 @@ test('the encoded Claude directory decodes only against the real filesystem', (t // A project whose directory is GONE cannot be recovered — stated as null, not // guessed at, which is what makes `everSeen` a floor rather than a fiction. assert.equal(decodeClaudeProjectDir(`${root.split(path.sep).join('-')}-vanished`), null); +}); + +// The decoder's refusal contract, asserted on EVERY platform. On Windows this is +// the whole of its observable behaviour: the encoding it decodes is `/`-rooted, +// a Windows path carries a drive prefix instead, and the decoder documents that +// as undecodable. The test above builds a `/`-rooted encoding Windows never +// produces, so running it there tested the platform rather than the decoder. +test('a name that is not a POSIX-absolute encoding is refused, on every platform', () => { assert.equal(decodeClaudeProjectDir('not-absolute'), null); assert.equal(decodeClaudeProjectDir(''), null); + assert.equal(decodeClaudeProjectDir(null), null); + assert.equal(decodeClaudeProjectDir('C:-Users-me-proj'), null, 'a Windows drive prefix is not decodable'); }); test('one project touched by two hosts is ONE project, resolved through symlinks', (t) => { @@ -209,7 +231,10 @@ test('an unreadable transcript makes the counts a floor, never a smaller total', assert.equal(section.complete, false); }); -test('a group with no cwd anywhere is recovered from its encoded directory name', (t) => { +// POSIX-only for the same reason as the decode test: the encoded directory +// name is built with path.sep, which on Windows yields a drive-prefixed form +// the decoder refuses by design, so the recovery it asserts cannot happen there. +test('a group with no cwd anywhere is recovered from its encoded directory name', { skip: POSIX_ONLY }, (t) => { const root = fixture(t, 'recover'); const project = path.join(root, 'quiet-project'); fs.mkdirSync(project, { recursive: true }); diff --git a/tests/kit/footprint-stack.test.mjs b/tests/kit/footprint-stack.test.mjs index 9a845a0..f6702e7 100644 --- a/tests/kit/footprint-stack.test.mjs +++ b/tests/kit/footprint-stack.test.mjs @@ -37,7 +37,13 @@ import { } from '../../src/lib/footprint/stack-detect.mjs'; function fixture(t, name) { - const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-stack-${name}-`))); + // realpathSync.NATIVE, matching what the collectors canonicalise with. The JS + // realpath leaves a Windows 8.3 short name alone (C:\Users\RUNNER~1\...) while the + // native one resolves it to the long form the code under test produces + // (C:\Users\runneradmin\...). Same directory, two spellings — and every path + // assertion in this file compared one against the other on Windows only. + const real = fs.realpathSync.native ?? fs.realpathSync; + const dir = real(fs.mkdtempSync(path.join(os.tmpdir(), `ak-fp-stack-${name}-`))); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); return dir; } From ea160f6bfc83637547ce3ef1c09b5d0fab75d7c4 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Sat, 8 Aug 2026 09:59:31 -0700 Subject: [PATCH 18/19] fix(footprint): the cloud-placeholder rule condemned every file on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fs.Stats.blocks` is a POSIX field. On win32 Node reports it as 0 for every file, and `isCloudPlaceholder` read that as "a provider has evicted this" — so every file was treated as dataless. Manifests were never queued, every source file was skipped, and a scan returned no lines, no dependencies and no stack. Eight of the thirteen Windows CI failures were this one gate; two independent symptoms pinned it (`manifestsRead` 0 where 2 were present, and a skip count one higher than the fixture's only binary file). win32 is excluded from the heuristic. The cost is real and stated in the code: Windows is where OneDrive Files On-Demand actually lives, so it is the platform that most needs this check and the one platform that cannot have it — detecting a placeholder there means FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS, which fs.Stats does not surface. A Windows placeholder is therefore opened and may block, exactly as before the heuristic existed. Reading one file slowly is recoverable; measuring nothing at all is not. `platform` is a parameter rather than a direct process.platform read, and the predicate is exported, so the win32 branch is testable from any machine. The end-to-end test shims lstatSync to report zero blocks — without that it would pass with or without the fix, since real POSIX files have blocks — and an anti-vacuity test asserts the damage is still reproducible on POSIX, so the guard cannot quietly stop being load-bearing. --- src/lib/footprint/stack-detect.mjs | 32 ++++++++++++--- tests/kit/footprint-stack.test.mjs | 65 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/lib/footprint/stack-detect.mjs b/src/lib/footprint/stack-detect.mjs index 5f7b290..5918ba2 100644 --- a/src/lib/footprint/stack-detect.mjs +++ b/src/lib/footprint/stack-detect.mjs @@ -131,10 +131,30 @@ function tailKey(lowerName) { * which is invariant 3's "unknown, never 0" rather than a fabricated zero. * * Only an explicit 0 counts. `undefined` means the stat did not carry blocks (a - * test shim, a platform that omits it), and guessing from a missing field would - * skip real files. Sparse files also report fewer blocks than their size implies - * but never zero-with-content, so they are unaffected. */ -const isCloudPlaceholder = (bytes, blocks) => blocks === 0 && bytes > 0; + * test shim), and guessing from a missing field would skip real files. Sparse + * files also report fewer blocks than their size implies but never + * zero-with-content, so they are unaffected. + * + * WINDOWS IS EXCLUDED, and not as a nicety. `fs.Stats.blocks` is a POSIX field; + * on win32 Node reports it as 0 for every file, content or not. The rule then + * reads EVERY file as evicted — manifests are never queued and every source + * file is skipped, so a scan returns no lines, no dependencies and no stack at + * all. Two independent CI symptoms pinned it: `manifestsRead` 0 where 2 were + * present, and a skip count one higher than the fixture's only binary file. + * + * The cost of the exclusion is real and worth stating: Windows is where OneDrive + * Files On-Demand actually lives, so it is the platform that most needs this + * check and the one platform that cannot have it. Detecting a placeholder there + * means reading FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS, which `fs.Stats` does not + * surface — so a Windows placeholder is opened and may block, exactly as it did + * before this heuristic existed. Reading one file slowly is recoverable; + * measuring nothing at all is not. + * + * `platform` is a parameter rather than a direct `process.platform` read so the + * win32 branch is testable from any machine. Pure. + * @param {number} bytes @param {number|undefined} blocks @param {string} platform */ +export const isCloudPlaceholder = (bytes, blocks, platform = process.platform) => + platform !== 'win32' && blocks === 0 && bytes > 0; /** Count newlines in one file through a fixed buffer. Returns null when the file * is binary (a NUL byte in the first chunk) or unreadable — never 0, which would @@ -388,6 +408,7 @@ function unmeasured(reason, asOf) { * @param {string} root the project's directory * @param {{ walk?: Function, limits?: object, maxDepth?: number, * manifestDepth?: number, signatureDepth?: number, manifests?: boolean, + * platform?: string, * asOf?: number|null, fsImpl?: typeof fs }} [options] * @returns {object} `{ registryVersion, languages[{id,name,ecosystem,colorSlot,lines, * files}], totalLines: Measurement, stack[{id,kind,name,ecosystem,via}], @@ -401,6 +422,7 @@ export function detectStack(root, { manifestDepth = MANIFEST_MAX_DEPTH, signatureDepth = SIGNATURE_MAX_DEPTH, manifests: readManifests = true, + platform = process.platform, asOf = null, fsImpl = fs, } = {}) { @@ -431,7 +453,7 @@ export function detectStack(root, { acceptFile: (name) => !EXCLUDED_FILES.has(name), onFile: ({ file, name, bytes, blocks, depth }) => { const lower = name.toLowerCase(); - const placeholder = isCloudPlaceholder(bytes, blocks); + const placeholder = isCloudPlaceholder(bytes, blocks, platform); if (depth <= signatureDepth) { note(seenFiles, lower); note(seenPaths, rel(root, file)); } // A placeholder manifest is still evidence the project HAS that manifest — // its name was noted above. Only its contents are out of reach, so it is diff --git a/tests/kit/footprint-stack.test.mjs b/tests/kit/footprint-stack.test.mjs index f6702e7..f9f7659 100644 --- a/tests/kit/footprint-stack.test.mjs +++ b/tests/kit/footprint-stack.test.mjs @@ -34,6 +34,7 @@ import { import { EXCLUDED_DIRS, EXCLUDED_FILES, MANIFEST_MAX_DEPTH, STACK_EXCLUSIONS, detectStack, parseManifestDependencies, stackProvenance, + isCloudPlaceholder, } from '../../src/lib/footprint/stack-detect.mjs'; function fixture(t, name) { @@ -392,3 +393,67 @@ test('the exclusions and the provenance travel with the figure', (t) => { assert.equal(none.unrecognizedExtensions, null); assert.equal(none.nonSourceFiles, null); }); + +// ── cloud placeholders, and the platform that cannot detect them ──────────── +// A dataless file (real size, zero allocated blocks) must never be opened: the +// read blocks in the kernel until the provider faults the bytes back, which +// with the provider offline is forever. But `blocks` is a POSIX field, and on +// win32 Node reports 0 for EVERY file — so the same rule there condemns the +// whole scan. This is the regression that took Windows CI down: 8 failures, +// all of them "measured nothing". + +test('a dataless file is a placeholder on POSIX and is never opened', () => { + assert.equal(isCloudPlaceholder(4096, 0, 'darwin'), true); + assert.equal(isCloudPlaceholder(4096, 0, 'linux'), true); +}); + +test('on win32 a zero block count is NOT evidence of a placeholder', () => { + // fs.Stats.blocks is POSIX-only; win32 reports 0 for every file. Reading the + // rule literally there skipped every source file and queued no manifest, so a + // scan returned no lines, no dependencies and no stack. + assert.equal(isCloudPlaceholder(4096, 0, 'win32'), false); + assert.equal(isCloudPlaceholder(1, 0, 'win32'), false); +}); + +test('a real file, an empty file and a missing blocks field are never placeholders', () => { + assert.equal(isCloudPlaceholder(4096, 8, 'darwin'), false, 'allocated blocks means real content'); + assert.equal(isCloudPlaceholder(0, 0, 'darwin'), false, 'an empty file legitimately has no blocks'); + assert.equal(isCloudPlaceholder(4096, undefined, 'darwin'), false, + 'a stat that did not carry blocks is unknown, and guessing would skip real files'); +}); + +/** A stat shim that reports zero allocated blocks for every file — what win32 + * actually does, reproduced so the guard is testable from any machine. Without + * this the platform argument changes nothing on POSIX (real files have blocks) + * and the test below would pass with or without the fix. */ +function zeroBlocksFs() { + // The walk stats with lstatSync, so that is the call to shim. The returned + // object must keep its prototype: the walk asks it isDirectory()/isFile(). + const zero = (st) => Object.assign(Object.create(Object.getPrototypeOf(st)), st, { blocks: 0 }); + return { ...fs, lstatSync: (target, opts) => zero(fs.lstatSync(target, opts)) }; +} + +test('with zero blocks reported, POSIX skips everything — the behaviour being guarded against', (t) => { + const root = fixture(t, 'zero-blocks-posix'); + write(root, 'real.js', 'a\nb\n'); + write(root, 'package.json', JSON.stringify({ dependencies: { 'totally-unknown-lib': '1.0.0' } })); + const out = detectStack(root, { asOf: 1, platform: 'darwin', fsImpl: zeroBlocksFs() }); + // Anti-vacuity: this is the exact damage seen on Windows CI. If this ever + // stops holding, the test below proves nothing. + assert.equal(out.manifests.length, 0, 'every file read as a placeholder — no manifest queued'); + assert.ok((out.languages.find((r) => r.id === 'javascript')?.lines ?? 0) === 0, 'and no lines counted'); +}); + +test('a win32 scan still counts lines, reads manifests and skips only the binary', (t) => { + const root = fixture(t, 'win32-blocks'); + write(root, 'real.js', 'a\nb\n'); + write(root, 'package.json', JSON.stringify({ dependencies: { 'totally-unknown-lib': '1.0.0' } })); + fs.writeFileSync(path.join(root, 'blob.js'), Buffer.from([0x61, 0x00, 0x62, 0x0a])); + + const out = detectStack(root, { asOf: 1, platform: 'win32', fsImpl: zeroBlocksFs() }); + assert.equal(out.skipped, 1, 'only the binary is skipped — not every file'); + assert.equal(out.languages.find((row) => row.id === 'javascript').lines, 2); + assert.equal(out.manifests.length, 1, 'the manifest was queued and read'); + assert.ok(out.unrecognized.dependencies.some((r) => r.name === 'totally-unknown-lib'), + 'and its declarations reached the unrecognized tail'); +}); From 7fb0c4efca1771306b9374c808af95951c86f481 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Sat, 8 Aug 2026 10:08:36 -0700 Subject: [PATCH 19/19] fix(footprint): the same POSIX-blocks assumption in two more places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit fixed the cloud-placeholder rule in stack-detect but missed a hand-rolled DUPLICATE of it in projectRemote, which is why Windows CI went 13 → 8 → 2 rather than to zero. Both remaining failures were that copy: `st.blocks === 0 && st.size > 0` on a .git/config, which on win32 is true for every file, so every project reported status 'unknown' with no remote URL. It now calls the shared exported predicate instead of restating the rule — the duplication is what let the first fix look complete. A third instance was latent rather than failing. measureAllocated derives allocated bytes from per-file `blocks` and falls back to apparent size only when the count is not finite. On win32 the count IS finite — it is 0 — so an entire machine's allocated total came out as zero bytes beside a correct apparent total. Its own doc comment already promised the Windows fallback this commit actually implements. No test covered it, so CI was green on a figure that would have been wrong on every Windows machine. Each fix carries an anti-vacuity assertion: the identical zero-blocks input on POSIX must still be read as a placeholder, and must still allocate zero. Without those the platform argument could stop being load-bearing and the tests would keep passing. --- src/lib/footprint/consumers.mjs | 25 ++++++++---- src/lib/footprint/projects.mjs | 12 ++++-- tests/kit/footprint-collectors.test.mjs | 53 +++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/src/lib/footprint/consumers.mjs b/src/lib/footprint/consumers.mjs index 1934a72..5a4d036 100644 --- a/src/lib/footprint/consumers.mjs +++ b/src/lib/footprint/consumers.mjs @@ -536,11 +536,14 @@ function measureFamily(desc, { walk, limits, asOf, fsImpl }) { * figure larger than the disk at the top of it. * * The walker still owns the traversal; the extra lstat per file is what buys - * `blocks`, which `onFile` does not carry. A platform that reports no block - * count (Windows) falls back to that file's apparent size, so the row degrades - * to the ordinary basis rather than to zero. + * `blocks`, which `onFile` does not carry. A platform that reports no usable + * block count (Windows) falls back to that file's apparent size, so the row + * degrades to the ordinary basis rather than to zero. + * + * `platform` is a parameter, not a `process.platform` read, so the win32 branch + * is exercisable from any machine. */ -function measureAllocated(desc, { walk, limits, asOf, fsImpl }) { +function measureAllocated(desc, { walk, limits, asOf, fsImpl, platform = process.platform }) { let allocated = 0; let apparent = 0; let files = 0; @@ -559,7 +562,14 @@ function measureAllocated(desc, { walk, limits, asOf, fsImpl }) { // an inode); only a platform that reports no block count at all falls // back to apparent size, and that fallback is counted so the row can say // its basis is mixed. - if (Number.isFinite(blocks) && blocks >= 0) allocated += blocks * 512; + // + // win32 is exactly that platform. `blocks` is POSIX; Node reports 0 for + // every file there, which is finite and non-negative — so without this + // the allocated total came out as 0 bytes for an entire machine while the + // apparent total was right beside it. Estimating from apparent size is + // the honest degradation, and `estimated` is what makes the row say so. + const usable = Number.isFinite(blocks) && blocks >= 0 && platform !== 'win32'; + if (usable) allocated += blocks * 512; else { allocated += bytes; estimated += 1; } }, }); @@ -781,7 +791,7 @@ const rankValue = (row) => (hasValue(row.bytes) ? row.bytes.value : -1); * install?: object|null, projects?: Array|null, * includeProjectTrees?: boolean, topN?: number, * extraRoots?: Array, - * fsImpl?: typeof fs, + * fsImpl?: typeof fs, platform?: string, * }} [options] `roots` replaces the registry outright (tests, narrowed scans); * `extraRoots` adds to it. `install` and `projects` are already-collected * sections whose figures are adopted rather than re-walked. @@ -804,9 +814,10 @@ export function collectConsumers({ topN = CONSUMER_TOP_N, extraRoots = [], fsImpl = fs, + platform = process.platform, } = {}) { const asOf = now(); - const ctx = { walk, limits: { ...CONSUMER_WALK_LIMITS, ...limits }, asOf, fsImpl }; + const ctx = { walk, limits: { ...CONSUMER_WALK_LIMITS, ...limits }, asOf, fsImpl, platform }; const candidates = Array.isArray(projects) ? projects : []; const descriptors = assignContainment(mergeDescriptors([ diff --git a/src/lib/footprint/projects.mjs b/src/lib/footprint/projects.mjs index 708eb1c..dfe4976 100644 --- a/src/lib/footprint/projects.mjs +++ b/src/lib/footprint/projects.mjs @@ -39,7 +39,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { parseRepoSlug } from '../admin-collect.mjs'; import { discoverProjectSources } from './project-sources.mjs'; -import { detectStack, STACK_EXCLUSIONS } from './stack-detect.mjs'; +import { detectStack, isCloudPlaceholder, STACK_EXCLUSIONS } from './stack-detect.mjs'; import { STACK_REGISTRY_VERSION } from './stack-registry.mjs'; import { walkTree, rootMeasurements, measured, statNode, UNKNOWN, unknown, sumMeasurements, @@ -155,7 +155,7 @@ export function describeRemote(rawUrl, name = 'origin') { * no configured remote is an explicit `local-only`; an unreadable `.git/config` * is `unknown` with its errno, not a silent local-only. */ -export function projectRemote(projectPath, { fsImpl = fs } = {}) { +export function projectRemote(projectPath, { fsImpl = fs, platform = process.platform } = {}) { const configFile = path.join(projectPath, '.git', 'config'); let source; // Stat before read: a cloud provider's evicted placeholder (real size, zero @@ -164,7 +164,13 @@ export function projectRemote(projectPath, { fsImpl = fs } = {}) { // honest answer — inventing `local-only` would claim this repo has no remote. try { const st = fsImpl.lstatSync(configFile); - if (st.blocks === 0 && st.size > 0) { + // The SHARED predicate, not a second copy of the rule. This was a hand-rolled + // duplicate of stack-detect's `blocks === 0 && size > 0`, and when that one + // was corrected for Windows — where `fs.Stats.blocks` is a POSIX field Node + // reports as 0 for every file — this copy was missed and went on reporting + // every .git/config as an unmaterialized placeholder, so every project on + // Windows had status 'unknown' and no remote URL. + if (isCloudPlaceholder(st.size, st.blocks, platform)) { return { status: 'unknown', name: null, raw: null, hostname: null, host: null, slug: null, webUrl: null, reason: 'cloud placeholder (not materialized)' }; } } catch { /* the read below reports the errno; one stat failure decides nothing */ } diff --git a/tests/kit/footprint-collectors.test.mjs b/tests/kit/footprint-collectors.test.mjs index 9076658..7ac7101 100644 --- a/tests/kit/footprint-collectors.test.mjs +++ b/tests/kit/footprint-collectors.test.mjs @@ -26,6 +26,7 @@ import { parseGitRemote, projectRemote, LOC_EXCLUSIONS, } from '../../src/lib/footprint/projects.mjs'; import { collectCatalog, tomlTableNames } from '../../src/lib/footprint/catalog.mjs'; +import { collectConsumers } from '../../src/lib/footprint/consumers.mjs'; const DAY = 86_400_000; @@ -1069,3 +1070,55 @@ test('an undecodable name says WHICH reason — deleted, or never encodable', () assert.equal(win.projectResolved, false); assert.equal(win.projectReason, 'encoding', 'nothing was deleted — the name was never decodable'); }); + +// ── the POSIX `blocks` field, and the platform that does not have it ──────── +// `fs.Stats.blocks` is POSIX. On win32 Node reports 0 for every file, and two +// separate places read a zero as meaning something it does not. Both took +// Windows CI down or would have silently under-reported it, so both are pinned +// here rather than only at their shared predicate. + +/** lstat that reports zero allocated blocks for every file — what win32 does. */ +function zeroBlocksFs(root) { + const base = fixtureFs(root); + const zero = (st) => Object.assign(Object.create(Object.getPrototypeOf(st)), st, { blocks: 0 }); + return { ...base, lstatSync: (target) => zero(base.lstatSync(target)) }; +} + +test('a git config reporting zero blocks is still read, not called a placeholder', (t) => { + const root = fixture(t, 'remote-zero-blocks'); + const repo = path.join(root, 'repo'); + write(path.join(repo, '.git', 'config'), + '[remote "origin"]\n\turl = https://github.com/pacphi/agentic-kit.git\n'); + + // The bug: every .git/config on Windows stated as an unmaterialized cloud + // placeholder, so every project reported status 'unknown' and no remote. + const out = projectRemote(repo, { fsImpl: zeroBlocksFs(root), platform: 'win32' }); + assert.equal(out.status, 'linked', `zero blocks is not evidence of eviction; got ${out.reason}`); + assert.equal(out.webUrl, 'https://github.com/pacphi/agentic-kit'); + + // Anti-vacuity: the identical input on POSIX IS an evicted placeholder, so the + // platform argument is doing the work rather than the shim being ignored. + const posix = projectRemote(repo, { fsImpl: zeroBlocksFs(root), platform: 'darwin' }); + assert.equal(posix.status, 'unknown'); + assert.match(posix.reason, /placeholder/); +}); + +test('allocated size falls back to apparent size where blocks are unusable', (t) => { + const root = fixture(t, 'alloc-zero-blocks'); + const dir = path.join(root, 'store'); + const bytes = write(path.join(dir, 'a.bin'), 'x'.repeat(4096)); + + const desc = { id: 'store', label: 'store', path: dir, allocation: 'blocks' }; + const run = (platform) => collectConsumers({ + roots: [desc], now: () => 1, fsImpl: zeroBlocksFs(root), platform, + }).rows.find((r) => r.id === 'store'); + + // Without the fallback this is 0 bytes for an entire machine, sitting next to + // a correct apparent total — a measured zero that is not true. + const win = run('win32'); + assert.ok(win.bytes.value >= bytes, `allocated collapsed to ${win.bytes.value}`); + + // Anti-vacuity: on POSIX a genuine zero-block file really does allocate + // nothing, so the platform argument is what changed the answer. + assert.equal(run('darwin').bytes.value, 0); +});