diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 02bad6e..46e0240 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -178,13 +178,18 @@ Defaults (all overridable; your edits are marked `custom` and never re-seeded): **Known-good model choices** (verified 2026-07; any model your host CLI accepts also works): +> **Per-token price ≠ per-task cost.** A model that needs more agentic turns costs more +> per task at the same per-token price. On subscription (`claude-code` oauth) billing the +> marginal dollar cost is $0 either way, and the extra turns are paid in wall-clock and +> quota instead — so read every note below on the turns axis, not only the price axis. + | Host | Model | When to use | |---|---|---| -| claude | `claude-opus-5` | new top Opus — ~2× Opus 4.8 at the same price; premium reasoning default | -| claude | `claude-sonnet-5` | near-Opus at lower cost — review, spec, release | +| claude | `claude-opus-5` | new top Opus — same per-token price as 4.8, but ~2–3× the agentic turns on routine work; earns it at the hard end | +| claude | `claude-sonnet-5` | near-Opus capability at a lower per-token price — review, spec, release | | claude | `claude-fable-5` | top capability (Mythos-class, above Opus 5) — hardest problems | -| claude | `claude-haiku-4-5-20251001` | cheap/fast — high-volume mechanical | -| claude | `claude-opus-4-8` | prior Opus generation — same price as opus-5, kept for pinned configs | +| claude | `claude-haiku-4-5-20251001` | cheap/fast — high-volume mechanical work | +| claude | `claude-opus-4-8` | prior Opus generation — same per-token price, roughly half the turns on routine work | | codex | `gpt-5.4` | coding + reasoning + agentic — recommended execution default | | codex | `gpt-5.6-sol` | newest line; first-class max reasoning effort | | codex | `gpt-5.3-codex` | pure coding-tuned — mechanical implementation & docs | @@ -192,8 +197,9 @@ Defaults (all overridable; your edits are marked `custom` and never re-seeded): > **Where Opus 5 sits** ([announcement](https://www.anthropic.com/news/claude-opus-5), July 2026): > same $5/$25 per-Mtok pricing as Opus 4.8 with roughly double the Frontier-Bench -> performance, so it strictly supersedes 4.8 as the reasoning-tier default — a capability -> tier above Opus 4.8 at no added cost. It is **not** Mythos-class: `claude-fable-5` +> performance, which is why it is the reasoning-tier default. That parity is **per token**: +> measured end-to-end it takes 2–3.4× the agentic turns on routine work, so per task it is +> the more expensive arm there and 4.8 remains a defensible pin. It is **not** Mythos-class: `claude-fable-5` > remains the flagship tier. Opus 5 lands within ~0.5% of Fable on coding/agentic > benchmarks at about half the cost per task, but stays behind the Mythos-class models on > frontier domains. Rule of thumb: `claude-opus-5` is the premium default; diff --git a/src/commands/status.mjs b/src/commands/status.mjs index 9208de0..7abb17a 100644 --- a/src/commands/status.mjs +++ b/src/commands/status.mjs @@ -8,7 +8,7 @@ import { loadRing, detectRegression } from '../lib/health-history.mjs'; import * as paths from '../lib/paths.mjs'; import { nativesStatus, rufloRuntimeNatives, dbPathPinStatus, aidefencePresent, securityPresent } from '../lib/natives.mjs'; import { scanNpxStale } from '../lib/npx.mjs'; -import { registrationStatus, codexMcpStatus, rufloCodexMcpStatus } from '../lib/mcp.mjs'; +import { registrationStatus, codexMcpStatus, rufloCodexMcpStatus, ruvectorRegistered } from '../lib/mcp.mjs'; import { listDaemons, staleDaemons } from '../lib/daemons.mjs'; import { scanRvf } from '../lib/rvf.mjs'; import { registry, syncBlocks, blocksForTarget, retiredForTarget, guidanceTargets } from '../lib/blocks.mjs'; @@ -19,8 +19,10 @@ import { drift as ruvnetBrainDrift, nightlyAgentPresent as rbNightlyPresent, NIG import { coherence as adbCoherence } from '../lib/agentdb.mjs'; import { readJson } from '../lib/settings.mjs'; import { have } from '../lib/exec.mjs'; -import { HOSTS, settingsTarget, isDefault, managedEnv, MANAGED_ENV_KEYS, hostInstallState, hostAuthState, bothHostsEnabled, aqeRouterFile, aqeSupportsAgentOverrides } from '../lib/providers.mjs'; -import { policyToAgentOverrides, routingSummary } from '../lib/routing.mjs'; +import { HOSTS, settingsTarget, isDefault, managedEnv, MANAGED_ENV_KEYS, hostInstallState, hostAuthState, bothHostsEnabled, aqeRouterFile, aqeSupportsAgentOverrides, credentialGaps } from '../lib/providers.mjs'; +import { policyToAgentOverrides, routingSummary, divergedRoutes } from '../lib/routing.mjs'; +import { qeCourtShipped, readQeCourtConfig, panelFromRouting, validatePanel, healJuryVendorCollision, UPSTREAM_JURY_VENDOR_ISSUE } from '../lib/qeCourt.mjs'; +import { drift as ruvectorDrift } from '../lib/ruvector.mjs'; export const options = { json: { type: 'boolean', default: false }, @@ -97,6 +99,35 @@ export async function collect({ pkgRoot, cwd = process.cwd() }) { } } + // ruvector — a global CLI users register as an MCP server BY HAND. ak manages + // its drift, never its presence or its registration. Unregistered → no row at + // all (same silence as codex-not-enabled): nudging a tool nobody opted into + // would be management by ambush. Registered but kit.json ruvector:false → an + // info row with NO fix, so sync never plans an upgrade the user turned off. + // + // Wording is deliberately "CLI": the registered command is typically + // `npx -y ruvector mcp start`, so upgrading the global package does not + // necessarily change what the MCP server executes. Claim only what is true. + if (ruvectorRegistered()) { + if (cfg.ruvector === false) { + rows.push(row('ruvector', 'info', 'ruvector MCP registered — CLI updates disabled (kit.json ruvector:false)')); + } else { + try { + const rv = await ruvectorDrift(); + if (rv.present && rv.outdated) { + rows.push(row('ruvector', 'warn', + `ruvector CLI ${rv.installed} installed, ${rv.latest} available`, 'sync upgrades the ruvector CLI')); + } else if (rv.present) { + rows.push(row('ruvector', 'ok', `ruvector CLI ${rv.installed}${rv.latest ? ' (latest)' : ''} (MCP registered, user scope)`)); + } else { + rows.push(row('ruvector', 'info', 'ruvector MCP registered but no global CLI installed (server runs via npx)')); + } + } catch (e) { + rows.push(row('ruvector', 'warn', `ruvector check unavailable: ${e.message}`)); + } + } + } + // self (the kit's own version — prerelease installs track the `next` tag) try { const s = await selfDrift({ pkgRoot }); @@ -348,6 +379,20 @@ export async function collect({ pkgRoot, cwd = process.cwd() }) { } else { rows.push(row('providers', 'ok', `wired: ${on}${chainStr} (${scope})`)); } + // Chain VIABILITY, separate from chain ORDER above: a chain in the right + // order whose rungs have no credential fails over into nothing (#54). Warn, + // not fail — the primary rung still works — and no `fix`, since only the + // user can supply a key. + if (chain.length) { + const gaps = credentialGaps(chain); + if (gaps.length) { + rows.push(row('providers', 'warn', + `aqe chain: ${chain.length - gaps.length}/${chain.length} rungs have credentials ` + + `(${gaps.map((g) => `${g.provider}: needs ${g.missing.join(', ')}`).join('; ')})`)); + } else { + rows.push(row('providers', 'ok', `aqe chain: ${chain.length}/${chain.length} rungs have credentials`)); + } + } } } catch (e) { rows.push(row('providers', 'warn', `provider check unavailable: ${e.message}`)); @@ -371,6 +416,20 @@ export async function collect({ pkgRoot, cwd = process.cwd() }) { if (drift) rows.push(row('routing', 'warn', `${base} — llm-config.json out of sync`, 'sync re-applies agentOverrides')); else rows.push(row('routing', 'ok', base)); } + // Seeded pins vs today's defaults. Deliberately `info` and deliberately + // "diverges from": which side wins is activity-dependent (a newer default + // can cost 2-3× the agentic turns on routine work), so a `warn` would push + // users to spend turns clearing a lint. No `fix` — sync must never + // auto-refresh a pin; `ak x provider refresh` is the opt-in path (#55). + const diverged = divergedRoutes(policy); + if (diverged.length) { + const pairs = [...new Set(diverged.flatMap((d) => [ + ...(d.modelDiverged ? [`${d.model} vs ${d.defaultModel}`] : []), + ...d.escalate.map((e) => `${e.model} vs ${e.defaultModel} (escalation)`), + ]))].join(', '); + rows.push(row('routing', 'info', + `${diverged.length} seeded route(s) diverge from current defaults (${pairs}) — ak x provider refresh`)); + } } } catch (e) { rows.push(row('routing', 'warn', `routing check unavailable: ${e.message}`)); @@ -479,6 +538,28 @@ export async function collect({ pkgRoot, cwd = process.cwd() }) { 'statusline is claude-only — codex has no command-backed statusline; its guidance ships via AGENTS.md')); } + // qe-court (ADR-124): TEMPORARY, remove once fixed upstream. agentic-qe's own + // shipped default config.json violates its own writerIsNeverJuror invariant + // (proffesor-for-testing/agentic-qe#576) — a brand-new project fails + // validation before any user touches the file. No-op unless aqe is new + // enough AND the skill has already created its config.json (ak never + // creates it) — same gate as `ak x provider status`'s read-only awareness. + if (qeCourtShipped()) { + const qcRoot = paths.repoRoot(cwd); + const qc = qcRoot ? readQeCourtConfig(qcRoot) : null; + if (qc) { + const violations = validatePanel(panelFromRouting(qc.routing), { minVendors: qc.options?.minDistinctVendors ?? 2 }); + if (violations.length) { + const fix = healJuryVendorCollision(qc.routing); + rows.push(row('qe-court', 'warn', + `qe-court panel invalid: ${violations.join(', ')}`, + fix ? `sync reassigns jury ${fix.from} → ${fix.to} (temporary until upstream fix lands: ${UPSTREAM_JURY_VENDOR_ISSUE})` : null)); + } else { + rows.push(row('qe-court', 'ok', 'qe-court panel valid (vendor-diverse, jury independent of writer)')); + } + } + } + return rows; } diff --git a/src/commands/sync.mjs b/src/commands/sync.mjs index 2d9545c..ff9d82c 100644 --- a/src/commands/sync.mjs +++ b/src/commands/sync.mjs @@ -11,6 +11,7 @@ import { listDaemons, staleDaemons, reap } from '../lib/daemons.mjs'; import { loadKitConfig, saveKitConfig } from '../lib/config.mjs'; import { HOSTS, applyHosts, applyProviders, hostInstallState, installHost, applyAqeRouter, seedDualRoutingIfDualHost, ensureCodexMcp, ensureRufloMcpInCodex, bothHostsEnabled } from '../lib/providers.mjs'; import { driftReport, selfDrift } from '../lib/versions.mjs'; +import { RUVECTOR_PKG, managed as ruvectorManaged } from '../lib/ruvector.mjs'; import { pruneNpxStale } from '../lib/npx.mjs'; import { nativesStatus, securityPresent } from '../lib/natives.mjs'; import { readJson } from '../lib/settings.mjs'; @@ -46,7 +47,7 @@ export async function run({ flags, pkgRoot }) { const cwd = process.cwd(); const rows = await collect({ pkgRoot, cwd }); const plan = rows.filter((r) => r.fix) - .filter((r) => !(flags['no-upgrade'] && (r.subsystem === 'versions' || r.subsystem === 'self' || r.subsystem === 'ruvnet-brain'))); + .filter((r) => !(flags['no-upgrade'] && ['versions', 'self', 'ruvnet-brain', 'ruvector'].includes(r.subsystem))); if (plan.length === 0) { ok('nothing to do — all subsystems healthy'); return 0; } @@ -75,6 +76,16 @@ export async function run({ flags, pkgRoot }) { if (subsystems.has('ruvnet-brain') && !flags['no-upgrade']) { await step('ruvnet-brain', () => heal.installRuvnetBrain({ force: true })); } + // ruvector: an unmanaged global users wire up as an MCP server by hand. Only + // ever UPGRADED — status emits no row (and so no plan entry) when it is absent, + // so this branch can never install it for someone who didn't opt in. + // The status row already gates on registration + opt-in (an unregistered or + // opted-out ruvector emits no `fix`, so it cannot reach this plan) — but this + // branch installs software globally, so it re-checks rather than trusting the + // plan to be the only guard. + if (subsystems.has('ruvector') && !flags['no-upgrade'] && ruvectorManaged(cfg)) { + await step('ruvector', () => heal.upgradePackage(RUVECTOR_PKG)); + } // The brain installer's own nightly self-updater (macOS LaunchAgent) bypasses // ak-managed updates — disabling it is a heal, not an upgrade, so it runs even // under --no-upgrade. Reversible: `npx ruvnet-brain --enable-nightly`. @@ -109,6 +120,12 @@ export async function run({ flags, pkgRoot }) { if (subsystems.has('aqe')) { report('rvf', heal.healRvf(paths.projectAqeDir(cwd))); } + // qe-court: TEMPORARY, remove once fixed upstream (agentic-qe#576) — see + // heal.healQeCourtPanel's doc comment. Only ever fires when status already + // found a fixable violation, so this never touches a valid or unfixable panel. + if (subsystems.has('qe-court')) { + report('qe-court', heal.healQeCourtPanel(cwd)); + } // agentdb: install/repin the standalone CLI to ruflo's bundled version so the // shared cognitive store stays coherent (harvest's write path depends on it). if (subsystems.has('agentdb') && cfg.agentdb !== false) { diff --git a/src/commands/x/provider.mjs b/src/commands/x/provider.mjs index 1e1d21f..5b5baad 100644 --- a/src/commands/x/provider.mjs +++ b/src/commands/x/provider.mjs @@ -11,21 +11,14 @@ import { undoProviders, hostInstallState, hostAuthState, installHost, applyAqeRouter, undoAqeRouter, bothHostsEnabled, DUAL_ROLE_TIP, JUDGE_BIAS_TIP, QE_COURT_TIP, suggestedFallbackFor, seedDualRoutingIfDualHost, printActivityRoutingTable, ensureCodexMcp, undoCodexMcp, - ensureRufloMcpInCodex, undoRufloMcpInCodex, + ensureRufloMcpInCodex, undoRufloMcpInCodex, detectAqeProviders, aqeProviderCredential, credentialGaps, fallbackSource, } from '../../lib/providers.mjs'; -import { parseRouteSpecs, formatModelHelp, PRIMARY_HOSTS, DEFAULT_PRIMARY_HOST } from '../../lib/routing.mjs'; +import { parseRouteSpecs, formatModelHelp, PRIMARY_HOSTS, DEFAULT_PRIMARY_HOST, divergedRoutes, refreshSeededRoutes, modelNote, ACTIVITIES } from '../../lib/routing.mjs'; import { loadKitConfig, saveKitConfig } from '../../lib/config.mjs'; -import { ok, warn, fail, info, dim, bold } from '../../lib/output.mjs'; -import { installedVersion, cmpVersions } from '../../lib/versions.mjs'; +import { ok, warn, fail, info, dim, bold, yellow } from '../../lib/output.mjs'; import { repoRoot } from '../../lib/paths.mjs'; import { writeJsonWithBackup } from '../../lib/settings.mjs'; -import { panelFromRouting, validatePanel, readQeCourtConfig, qeCourtConfigPath, vendorOf } from '../../lib/qeCourt.mjs'; - -const QE_COURT_MIN_VERSION = '3.13.0'; -const qeCourtShipped = () => { - const v = installedVersion('agentic-qe'); - return !!v && cmpVersions(v, QE_COURT_MIN_VERSION) >= 0; -}; +import { panelFromRouting, validatePanel, readQeCourtConfig, qeCourtConfigPath, vendorOf, qeCourtShipped } from '../../lib/qeCourt.mjs'; /** Print the dual-host guidance tips (role delegation, judge-bias, qe-court * cross-sell) once both hosts are enabled — shared by `pick()` and @@ -44,6 +37,7 @@ export const options = { 'aqe-fallback': { type: 'string' }, // 'claude-code:model1,model2;openai:gpt-5.6' ('none' clears) provider: { type: 'string' }, // csv of ruflo API providers, optional id:model (openai:gpt-5.6) route: { type: 'string', multiple: true }, // repeatable: 'activity:host[:model]' per-activity routing override + activity: { type: 'string' }, // refresh: csv of activities to re-seed (default = prompt) yes: { type: 'boolean', default: false }, json: { type: 'boolean', default: false }, }; @@ -62,6 +56,9 @@ persist to kit.json → idempotent heal. \`ak sync\` reapplies your choice. Subcommands: status (default) detected CLIs, aqe provider, ruflo providers, what's wired pick choose hosts / aqe provider / ruflo providers → persist → apply + refresh re-seed routes whose seeded pin diverges from the current defaults + (per-activity, opt-in; user pins are never touched, and \`ak sync\` + never does this for you) off reversible teardown (reset to claude-only; strip managed env keys) Options (pick, all optional — omit for interactive): @@ -84,6 +81,7 @@ Options (pick, all optional — omit for interactive): implementation, testing, review, security-scan, security-analysis, documentation, debugging, packaging, release + --activity refresh: which activities to re-seed (default: prompt) --yes accept defaults without prompting When both claude and codex hosts are enabled (and aqe ≥ 3.13.1), ak seeds a @@ -95,8 +93,14 @@ Examples: ak x provider show what's detected + wired + routing ak x provider pick --host claude,codex ak x provider pick --route 'testing:claude:claude-sonnet-5' + ak x provider refresh --activity architecture,design ak x provider off`; +/** Stamp provenance onto chain entries. 'suggested' = ak proposed it and the + * user pressed enter; 'user' = they typed it. Only 'suggested' entries are + * eligible for an offered refresh — a typed pin is intent (#55). */ +const stamp = (source) => (entries) => entries.map((e) => ({ ...e, source })); + /** Parse 'claude-code:m1,m2; openai:gpt-5.6' → [{provider, models:[…]}, …]. */ const parseFallback = (str) => str.split(';').map((s) => s.trim()).filter(Boolean).map((tok) => { const [provider, models] = tok.split(':'); @@ -110,8 +114,9 @@ export async function run({ flags, positionals }) { if (sub === 'status') return status({ flags, cwd }); if (sub === 'off') return off({ cwd }); if (sub === 'pick') return pick({ flags, cwd }); + if (sub === 'refresh') return refresh({ flags, cwd }); - fail(`unknown provider subcommand: ${sub} (status|pick|off)`); + fail(`unknown provider subcommand: ${sub} (status|pick|refresh|off)`); return 2; } @@ -149,12 +154,31 @@ async function status({ flags, cwd }) { console.log(` ${dim(AQE_BILLING_HINT)}`); const chain = cfg.providers.aqeFallback ?? []; if (chain.length) { - const rendered = chain.map((e) => `${e.provider}${e.models?.length ? `(${e.models.join(',')})` : dim('(no models)')}`).join(' → '); + const rendered = chain.map((e) => { + const cred = aqeProviderCredential(e.provider); + const models = e.models?.length ? `(${e.models.join(',')})` : dim('(no models)'); + return `${e.provider}${models}${cred.present ? '' : yellow(' ⚠ no credential')}`; + }).join(' → '); console.log(` ${dim('fallback chain:')} ${rendered} ${dim('· .agentic-qe/llm-config.json')}`); + for (const g of credentialGaps(chain)) { + warn(`fallback rung '${g.provider}' has no credential — needs ${g.missing.join(', ')} in the env; it will fail over into nothing`); + } } else { console.log(` ${dim('fallback chain: none (aqe auto-enables keyed providers)')}`); } + // Credential state for EVERY aqe provider type, so a provider that is + // credentialed on this machine (openrouter, say) is never invisible while an + // uncredentialed one is displayed as a configured fallback (#54). + const creds = detectAqeProviders(); + console.log(bold('\naqe provider credentials') + dim(' (keys read from env; never persisted)')); + for (const p of AQE_PROVIDER_TYPES) { + const c = creds[p]; + const state = c.present ? (c.billing === 'local' ? 'local' : c.billing === 'subscription' ? 'subscription' : 'key present') + : `no key ${dim(`(${c.missing.join(', ')})`)}`; + console.log(` ${p.padEnd(14)} ${state}${c.source && c.present && c.billing === 'metered' ? dim(` · ${c.source}`) : ''}`); + } + const cm = cfg.providers.models ?? []; console.log(bold('\nruflo LLM API providers') + dim(' (ruflo router; keys read from env)')); for (const p of API_PROVIDERS) { @@ -196,6 +220,60 @@ function printQeCourtStatus(cwd) { else ok('qe-court panel valid (vendor-diverse, jury independent of writer)'); } +/** Opt-in, per-activity re-seed of routes whose seeded pin diverges from the + * current defaults. Deliberately a separate command, never part of `ak sync`: + * sync is documented as idempotent reapplication of persisted choice, and the + * newer default is not uniformly better — on routine work it can cost 2-3× the + * agentic turns for the same result (#55). Only `source: 'seeded'` entries are + * eligible; a user pin survives even when named. */ +async function refresh({ flags, cwd }) { + const cfg = loadKitConfig(); + const policy = cfg.providers?.dualRouting ?? {}; + const diverged = divergedRoutes(policy); + if (!diverged.length) { ok('no seeded routes diverge from the current defaults'); return 0; } + + console.log(bold('seeded routes that diverge from current defaults')); + for (const d of diverged) { + const head = d.modelDiverged ? `${d.model} → ${d.defaultModel}` : d.model; + console.log(` ${d.activity.padEnd(18)} ${d.host.padEnd(7)} ${head}`); + for (const e of d.escalate) console.log(` ${dim('escalation:')} ${e.model} → ${e.defaultModel}`); + // The trade, not just the ids: choosing on a price axis while paying on a + // turns axis is the misreading this whole surface exists to prevent. + if (d.modelDiverged && d.currentNote) console.log(dim(` now: ${d.model} — ${d.currentNote}`)); + if (d.modelDiverged && d.defaultNote) console.log(dim(` default: ${d.defaultModel} — ${d.defaultNote}`)); + for (const e of d.escalate) { + const note = modelNote(e.defaultModel); + if (note) console.log(dim(` default: ${e.defaultModel} — ${note}`)); + } + } + + let picked; + if (flags.activity !== undefined) { + const want = flags.activity.split(',').map((s) => s.trim()).filter(Boolean); + for (const a of want.filter((a) => !ACTIVITIES.includes(a))) warn(`unknown activity '${a}' — ignored`); + picked = want.filter((a) => diverged.some((d) => d.activity === a)); + } else if (flags.yes) { + picked = diverged.map((d) => d.activity); + } else { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const ans = (await rl.question(`refresh which activities? (comma-separated, "all", blank = none) [${diverged.map((d) => d.activity).join(',')}]: `)).trim(); + rl.close(); + if (!ans) { info('nothing refreshed — routes left as they are'); return 0; } + picked = ans.toLowerCase() === 'all' + ? diverged.map((d) => d.activity) + : ans.split(',').map((s) => s.trim()).filter((a) => diverged.some((d) => d.activity === a)); + } + if (!picked.length) { info('nothing refreshed — routes left as they are'); return 0; } + + cfg.providers.dualRouting = refreshSeededRoutes(policy, { activities: picked }); + saveKitConfig(cfg); + ok(`refreshed ${picked.length} route(s): ${picked.join(', ')}`); + const router = applyAqeRouter(cfg, cwd); + (router.ok ? ok : warn)(`aqe router: ${router.detail}`); + printActivityRoutingTable(cfg); + return 0; +} + async function off({ cwd }) { const cfg = loadKitConfig(); const codexMcpManaged = cfg.providers?.codexMcp === 'ak'; @@ -263,7 +341,9 @@ async function pick({ flags, cwd }) { const hosts = await detectHosts(cwd); let enabled; let aqeProvider = cfg.providers.aqeProvider ?? null; - let aqeFallback = cfg.providers.aqeFallback ?? []; + // A legacy chain written before provenance existed reads as 'user': we cannot + // tell whether it was typed or accepted, so it must never be auto-touched. + let aqeFallback = (cfg.providers.aqeFallback ?? []).map((e) => ({ ...e, source: fallbackSource(e) })); let models = cfg.providers.models ?? []; const prevPrimary = cfg.providers.primaryHost ?? DEFAULT_PRIMARY_HOST; const oldPolicy = cfg.providers.dualRouting ?? {}; @@ -281,7 +361,7 @@ async function pick({ flags, cwd }) { } if (flags['aqe-fallback'] !== undefined) { const v = flags['aqe-fallback'].trim().toLowerCase(); - aqeFallback = (v === 'none' || v === '') ? [] : parseFallback(v); + aqeFallback = (v === 'none' || v === '') ? [] : stamp('user')(parseFallback(v)); } if (flags.provider !== undefined) models = parseModels(flags.provider); } else { @@ -298,7 +378,9 @@ async function pick({ flags, cwd }) { const fAns = (await rl.question( `aqe fallback chain, ordered (e.g. "claude-code:claude-opus-5; openai:gpt-5.6"${suggestion ? `, blank = use suggested [${suggestion}]` : ', blank = none'}): `, )).trim().toLowerCase(); - aqeFallback = fAns ? parseFallback(fAns) : (suggestion ? parseFallback(suggestion.toLowerCase()) : []); + aqeFallback = fAns + ? stamp('user')(parseFallback(fAns)) + : (suggestion ? stamp('suggested')(parseFallback(suggestion.toLowerCase())) : []); const provAns = (await rl.question('ruflo API-key providers to register (e.g. openai:gpt-5.6, blank to skip): ')).trim(); if (provAns) models = parseModels(provAns); rl.close(); @@ -335,6 +417,17 @@ async function pick({ flags, cwd }) { else if (!e.models.length) warn(`fallback entry '${e.provider}' has no models — aqe may skip it; add e.g. ${e.provider}:`); return okp; }); + // Tell the user AT ENTRY TIME that a rung is inert, and name what it needs — + // the failure otherwise surfaces at QE-run time, far from the config (#54). + const gaps = credentialGaps(aqeFallback); + for (const g of gaps) { + warn(`${g.provider}: no ${g.missing.join(' / ')} in env — this rung will fail over into nothing`); + } + if (gaps.length) { + const live = AQE_PROVIDER_TYPES.filter((p) => aqeProviderCredential(p).present + && !aqeFallback.some((e) => e.provider === p)); + if (live.length) info(`credentialed alternatives available now: ${live.join(', ')}`); + } cfg.providers = { hosts: { claude: enabled.includes('claude'), codex: enabled.includes('codex') }, diff --git a/src/lib/config.mjs b/src/lib/config.mjs index b86c933..5334fa5 100644 --- a/src/lib/config.mjs +++ b/src/lib/config.mjs @@ -9,6 +9,7 @@ const DEFAULTS = { aqe: true, // manage agentic-qe alongside ruflo agentdb: true, // manage the standalone agentdb CLI (harvest's write path), pinned to ruflo's bundled version ruvnetBrain: true, // install/manage the RuvNet Brain (offline KB + search_ruvnet MCP) + ruvector: true, // report drift for a globally-installed ruvector CLI (never installs it) security: true, // run the security verification surface by default harvest: false, // opt-in learning-write (`ak x harvest`); off = never runs writes health: { ring: [] }, // persisted stack-health snapshot ring (see health-history.mjs) diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index 3e83a54..f6914fc 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -27,6 +27,7 @@ import { fileURLToPath } from 'node:url'; import { execFile } from 'node:child_process'; import { driftReport, selfDrift } from './versions.mjs'; import { drift as ruvnetBrainDrift } from './ruvnet-brain.mjs'; +import { drift as ruvectorDrift, managed as ruvectorManaged } from './ruvector.mjs'; import { loadKitConfig } from './config.mjs'; import { resolveRoutes, routingSummary, ACTIVITIES, HOST_PROVIDER } from './routing.mjs'; import { renderPage } from './dashboard/page.mjs'; @@ -126,6 +127,9 @@ async function collectData({ cwd, fetchStatus }) { try { if (loadKitConfig().ruvnetBrain) drift = foldBrainDrift(drift, await ruvnetBrainDrift()); } catch { /* banner is best-effort — the subsystem card still carries the brain row */ } + try { + if (ruvectorManaged()) drift = foldRuvectorDrift(drift, await ruvectorDrift()); + } catch { /* banner is best-effort — the subsystem card still carries the ruvector row */ } } return { @@ -185,6 +189,20 @@ export function foldBrainDrift(drift, b) { }]; } +/** Fold the ruvector CLI into the npm drift array, same shape renderDrift wants. + * `r` is a src/lib/ruvector.mjs drift() result. Absent ruvector → array + * unchanged: ak reports its drift but never advertises installing it. + * Pure; exported for tests. */ +export function foldRuvectorDrift(drift, r) { + if (!r?.present) return drift; + return [...(drift ?? []), { + pkg: 'ruvector', + installed: r.installed, + latest: r.latest, + outdated: !!r.outdated, + }]; +} + // ───────────────────────────────────────────────────────────────────────────── // Usage tab plumbing (ADR-0009). Everything here is either a pure guard or a // thin adapter over usage-index.mjs — no parsing, no aggregation, no pricing. diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index 9e542cb..0208906 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -68,7 +68,8 @@ export const JS = ` var CAT={ hosts:"hosts", mcp:"hosts", "codex-mcp":"hosts", routing:"hosts", providers:"providers", - learning:"intel", "ruvnet-brain":"intel", "ruvnet-brain-nightly":"intel", aqe:"intel", agentdb:"intel" + learning:"intel", "ruvnet-brain":"intel", "ruvnet-brain-nightly":"intel", aqe:"intel", agentdb:"intel", + ruvector:"intel" }; function catOf(s){return CAT[s]||"runtime";} @@ -143,7 +144,7 @@ export const JS = ` // severity rank for rollups + triage sort; preferred order breaks ties. var RANK={fail:3,warn:2,ok:1,info:0,unknown:0}; - var PREF=["versions","self","natives","security","learning","providers","hosts","routing","mcp","codex-mcp","ruvnet-brain","ruvnet-brain-nightly","aqe","daemons","blocks","statusline","npx"]; + var PREF=["versions","self","natives","security","learning","providers","hosts","routing","mcp","codex-mcp","ruvnet-brain","ruvnet-brain-nightly","ruvector","aqe","daemons","blocks","statusline","npx"]; // Collapse rows into one group per subsystem (kills repeated labels); the // group's level is the worst of its rows. Sort worst-first, then by PREF. diff --git a/src/lib/dashboard/styles.mjs b/src/lib/dashboard/styles.mjs index 49fdd81..7ab850f 100644 --- a/src/lib/dashboard/styles.mjs +++ b/src/lib/dashboard/styles.mjs @@ -49,6 +49,7 @@ body{ /* ── token gate (dashboard requires a per-session token, same contract as admin — ADR-0014) — hides the rest of the page via body.gated below ── */ .gate{position:fixed;inset:0;z-index:50;display:flex;align-items:center;justify-content:center;background:var(--bg);padding:24px} +.gate[hidden]{display:none} .gate-card{max-width:460px;background:var(--panel);border:1px solid var(--line);border-radius:var(--r);padding:26px 28px} .gate-card p{font-size:14px;color:var(--ink-2);margin:0 0 14px;line-height:1.5} .gate-card code{font-family:var(--mono);color:var(--accent);font-size:12.5px} diff --git a/src/lib/heal.mjs b/src/lib/heal.mjs index 929aac2..88a246d 100644 --- a/src/lib/heal.mjs +++ b/src/lib/heal.mjs @@ -6,12 +6,14 @@ import fs from 'node:fs'; import path from 'node:path'; import { run } from './exec.mjs'; -import { rufloRoot, aqeRoot } from './paths.mjs'; +import { rufloRoot, aqeRoot, repoRoot } from './paths.mjs'; import { agentdbLocations, bsq3IsNative, bsq3Root, deriveBsq3Spec, selfSpecConflicts, rufloMemoryContexts, aidefencePresent } from './natives.mjs'; import { KIT_PKG } from './versions.mjs'; import { scanRvf, quarantine } from './rvf.mjs'; import { INSTALL_SPEC, INSTALL_ARGS, NIGHTLY_LABEL as RB_NIGHTLY_LABEL, nightlyAgentPlist as rbNightlyPlist, present as rbPresent, latestVersion as rbLatest, recordInstalledRelease as rbRecord } from './ruvnet-brain.mjs'; import { PKG as ADB_PKG, present as adbPresent, coherence as adbCoherence } from './agentdb.mjs'; +import { readQeCourtConfig, qeCourtConfigPath, healJuryVendorCollision, UPSTREAM_JURY_VENDOR_ISSUE } from './qeCourt.mjs'; +import { writeJsonWithBackup } from './settings.mjs'; // Packages whose install scripts must run for natives to build (npm >=11.17 // blocks them by default). Curated on the live 3.28/3.12.2 upgrade. @@ -149,6 +151,26 @@ export function healRvf(projectAqeDir) { return { ok: true, detail: removed.length ? `quarantined: ${removed.join(', ')}` : 'healthy' }; } +/** TEMPORARY (remove once fixed upstream): agentic-qe's own shipped default + * qe-court config.json violates its own writerIsNeverJuror invariant — see + * UPSTREAM_JURY_VENDOR_ISSUE in qeCourt.mjs. Auto-corrects ONLY the `jury` + * role, ONLY when the current config actually has the vendor collision, by + * reassigning it to an already-configured distinct-vendor provider. Never + * invents a vendor the project hasn't configured, never touches any other + * key. No-op (ok:true) when there's no project root or no config yet — ak + * never creates this file. */ +export function healQeCourtPanel(cwd = process.cwd()) { + const root = repoRoot(cwd); + if (!root) return { ok: true, detail: 'no project root — skipped' }; + const qc = readQeCourtConfig(root); + if (!qc) return { ok: true, detail: 'no qe-court config present — skipped' }; + const fix = healJuryVendorCollision(qc.routing); + if (!fix) return { ok: true, detail: 'qe-court panel already valid (or unfixable automatically)' }; + const routing = { ...qc.routing, jury: { ...qc.routing.jury, provider: fix.to } }; + writeJsonWithBackup(qeCourtConfigPath(root), { ...qc, routing }); + return { ok: true, detail: `jury ${fix.from} → ${fix.to} (temporary until upstream fix lands: ${UPSTREAM_JURY_VENDOR_ISSUE})` }; +} + /** Upgrade a global package to latest (with allow-scripts). */ export async function upgradePackage(pkg) { const r = await run('npm', ['install', '-g', `--allow-scripts=${ALLOW_SCRIPTS}`, `${pkg}@latest`], diff --git a/src/lib/mcp.mjs b/src/lib/mcp.mjs index f22a9ae..9bf305a 100644 --- a/src/lib/mcp.mjs +++ b/src/lib/mcp.mjs @@ -42,6 +42,15 @@ export function registrationStatus() { }; } +/** Is the standalone ruvector MCP server registered at user scope? Spawn-free + * read of ~/.claude.json (same seam as registrationStatus) — `claude mcp list` + * runs a live health check per server and has no stable output schema. ak NEVER + * registers this: user registration IS the opt-in signal that ak may keep the + * global CLI current. */ +export function ruvectorRegistered() { + return 'ruvector' in (readJson(claudeUserMcpPath(), {})?.mcpServers ?? {}); +} + /** * Project-scoped codex MCP (mcp__codex__codex) registration state. `ensureCodexMcp` * registers it via `claude mcp add codex -s project`, which persists to `.mcp.json` diff --git a/src/lib/providers.mjs b/src/lib/providers.mjs index f43bf89..f255575 100644 --- a/src/lib/providers.mjs +++ b/src/lib/providers.mjs @@ -31,7 +31,7 @@ import { readJson, writeJsonWithBackup } from './settings.mjs'; import { installedVersion, cmpVersions } from './versions.mjs'; import * as paths from './paths.mjs'; import { bold, dim, cyan } from './output.mjs'; -import { policyToAgentOverrides, seedDualRouting, resolveRoutes, routingSummary, ACTIVITIES, DEFAULT_PRIMARY_HOST, PRIMARY_HOSTS } from './routing.mjs'; +import { policyToAgentOverrides, seedDualRouting, resolveRoutes, routingSummary, divergedRoutes, ACTIVITIES, DEFAULT_PRIMARY_HOST, PRIMARY_HOSTS } from './routing.mjs'; import { HOST_ADAPTERS } from './hosts.mjs'; /** Frontier agent-CLI hosts. `pkg` is the npm global package; `enableEnv` is @@ -65,6 +65,86 @@ export const AQE_PROVIDER_TYPES = [ 'azure-openai', 'bedrock', 'cognitum', 'ollama', 'onnx', ]; +/** Credential descriptor per AQE_PROVIDER_TYPES member — the missing half of the + * chain validation: a rung whose provider has no usable credential is inert, and + * a type/shape check can't see that (#54). Env names are GROUNDED in aqe's own + * provider implementations (dist/shared/llm/providers/*.js `getApiKey`), not + * guessed; `also` are the extra vars a provider hard-requires beyond the key. + * host — credentialed by a frontier host's login rather than an env key + * local — runs locally, no credential ($0) + * Keyed by provider id so AQE_PROVIDER_TYPES and this map cannot diverge silently + * (a test asserts the two cover exactly the same set). */ +export const AQE_PROVIDER_CREDENTIALS = { + 'claude-code': { host: 'claude', billing: 'subscription' }, + // `codex` is deliberately absent from AQE_PROVIDER_TYPES (that list also gates + // AQE_LLM_PROVIDER validation, so widening it is a separate behavior change) — + // but routing's AQE_CONSTRUCTIBLE_PROVIDERS includes it and + // policyToAgentOverrides emits `provider: 'codex'`, so it needs a descriptor or + // those projected entries would be unverifiable. + codex: { host: 'codex', billing: 'subscription' }, + claude: { keyEnv: ['ANTHROPIC_API_KEY'], billing: 'metered' }, + openai: { keyEnv: ['OPENAI_API_KEY'], billing: 'metered' }, + gemini: { keyEnv: ['GOOGLE_AI_API_KEY', 'GEMINI_API_KEY', 'GOOGLE_API_KEY'], billing: 'metered' }, + openrouter: { keyEnv: ['OPENROUTER_API_KEY'], billing: 'metered' }, + 'azure-openai': { keyEnv: ['AZURE_OPENAI_API_KEY'], also: ['AZURE_OPENAI_ENDPOINT'], billing: 'metered' }, + bedrock: { keyEnv: ['AWS_ACCESS_KEY_ID'], also: ['AWS_SECRET_ACCESS_KEY'], billing: 'metered' }, + cognitum: { keyEnv: ['COGNITUM_API_KEY'], billing: 'metered' }, + ollama: { local: true, billing: 'local' }, + onnx: { local: true, billing: 'local' }, +}; + +/** Is this aqe provider actually usable right now? Returns + * {known, present, billing, source, missing[]}. `source` names the env var (or + * host login) that satisfied it; `missing` names what would satisfy it. Pure + * except for the injected env. */ +export function aqeProviderCredential(provider, { env = process.env, hostAuth = hostAuthState } = {}) { + const d = AQE_PROVIDER_CREDENTIALS[provider]; + if (!d) return { known: false, present: false, billing: 'unknown', source: null, missing: [] }; + if (d.host) { + const auth = hostAuth(d.host, { env }); + return { known: true, present: auth.mode !== 'none', billing: d.billing, source: auth.source, missing: auth.mode === 'none' ? [`${d.host} login`] : [] }; + } + if (d.local) return { known: true, present: true, billing: d.billing, source: 'local', missing: [] }; + const key = d.keyEnv.find((k) => !!env[k]); + const missingAlso = (d.also ?? []).filter((k) => !env[k]); + return { + known: true, + present: !!key && missingAlso.length === 0, + billing: d.billing, + source: key ?? null, + missing: [...(key ? [] : [d.keyEnv.join(' | ')]), ...missingAlso], + }; +} + +/** Chain rungs that cannot execute — [{provider, missing}] in chain order. The + * shared basis for the write-time warning, the pick-time warning, and the + * `ak status` viability row, so the three can never disagree. + * @param {Array<{provider?: string}>} [chain] + * @param {{ env?: NodeJS.ProcessEnv, hostAuth?: typeof hostAuthState }} [opts] + * @returns {Array<{provider: string, missing: string[]}>} */ +export function credentialGaps(chain = [], { env = process.env, hostAuth } = {}) { + return chain + .filter((e) => e?.provider) + .map((e) => ({ provider: e.provider, ...aqeProviderCredential(e.provider, { env, ...(hostAuth ? { hostAuth } : {}) }) })) + // `known === false` is UNVERIFIABLE, not uncredentialed: an unrecognized + // provider must never be reported as "no credential" — that would be ak + // asserting a fact about a provider it has no descriptor for. The separate + // unknown-provider path (applyAqeRouter's type filter) already handles it. + .filter((c) => c.known && !c.present) + .map((c) => ({ provider: c.provider, missing: c.missing })); +} + +/** Credential state for every aqe provider type — what `ak x provider status` + * renders, so a credentialed provider (e.g. openrouter) is never invisible. */ +export function detectAqeProviders({ env = process.env } = {}) { + return Object.fromEntries(AQE_PROVIDER_TYPES.map((p) => [p, aqeProviderCredential(p, { env })])); +} + +/** Provenance for an aqeFallback entry. A legacy entry written before stamping + * is treated as 'user' — never auto-touched, since we cannot tell whether the + * user typed it or accepted a suggestion (#55). */ +export const fallbackSource = (entry) => entry?.source ?? 'user'; + /** Every env key this module owns — the reversible surface for `off`/undo. */ export const MANAGED_ENV_KEYS = [ 'ENABLE_CLAUDE_CODE', 'ENABLE_CODEX', 'ENABLE_GEMINI_MCP', @@ -297,7 +377,12 @@ export function applyAqeRouter(cfg, cwd = process.cwd()) { for (const e of valid) next.providers[e.provider] = { ...(existing.providers?.[e.provider] ?? {}), enabled: true }; next.fallbackChain = buildChain(valid); const emptyModels = valid.filter((e) => !e.models || e.models.length === 0).map((e) => e.provider); - details.push(`chain: ${valid.map((e) => e.provider).join(' → ')}${emptyModels.length ? ` (⚠ no models for: ${emptyModels.join(', ')})` : ''}`); + // Warn, never refuse: the user may export the key later, and silently + // dropping a rung is worse than writing one that is currently inert (#54). + const gaps = credentialGaps(valid); + details.push(`chain: ${valid.map((e) => e.provider).join(' → ')}` + + (emptyModels.length ? ` (⚠ no models for: ${emptyModels.join(', ')})` : '') + + (gaps.length ? ` (⚠ no credential for: ${gaps.map((g) => `${g.provider} — needs ${g.missing.join(', ')}`).join('; ')})` : '')); wrote = true; } } @@ -393,12 +478,23 @@ export function formatRoutingTable(cfg) { const s = routingSummary(policy); const lines = [bold('\nper-activity routing') + dim(` (${s.byHost.claude ?? 0} claude · ${s.byHost.codex ?? 0} codex · ${s.custom} custom · .agentic-qe/llm-config.json)`)]; + // "diverges from", never "stale"/"outdated"/"superseded": the pinned model is + // sometimes the better choice for an activity, so the wording must present a + // decision rather than a lag (#55). + const diverged = Object.fromEntries(divergedRoutes(policy).map((d) => [d.activity, d])); for (const act of ACTIVITIES) { const r = routes[act]; const src = r.source === 'user' ? cyan('custom') : dim(r.source); const esc = r.escalate?.length ? dim(` ↑ ${r.escalate.map((e) => e.host).join('→')}`) : ''; const tag = r.akOriginated ? dim(' [ak]') : ''; - lines.push(` ${act.padEnd(18)} ${r.host.padEnd(7)} ${(r.model ?? '').padEnd(24)} ${src}${tag}${esc}`); + const d = diverged[act]; + const div = d + ? dim(` diverges from default ${d.modelDiverged ? d.defaultModel : d.escalate.map((e) => `↑${e.defaultModel}`).join(',')}`) + : ''; + lines.push(` ${act.padEnd(18)} ${r.host.padEnd(7)} ${(r.model ?? '').padEnd(24)} ${src}${tag}${esc}${div}`); + } + if (Object.keys(diverged).length) { + lines.push(dim(` ${Object.keys(diverged).length} seeded route(s) diverge from current defaults — review with: ak x provider refresh`)); } return lines.join('\n'); } diff --git a/src/lib/qeCourt.mjs b/src/lib/qeCourt.mjs index 6dc67a4..a31954e 100644 --- a/src/lib/qeCourt.mjs +++ b/src/lib/qeCourt.mjs @@ -10,6 +10,15 @@ // file and never touches any other key in it. import path from 'node:path'; import { readJson } from './settings.mjs'; +import { installedVersion, cmpVersions } from './versions.mjs'; + +const QE_COURT_MIN_VERSION = '3.13.0'; + +/** Is the installed agentic-qe new enough to have shipped qe-court (ADR-124)? */ +export function qeCourtShipped() { + const v = installedVersion('agentic-qe'); + return !!v && cmpVersions(v, QE_COURT_MIN_VERSION) >= 0; +} /** Map a provider id to its coarse vendor — ported from qe-court's referee.js. */ export function vendorOf(providerId) { @@ -47,6 +56,35 @@ export function validatePanel(panel, policy = {}) { return violations; } +// TEMPORARY (remove once fixed upstream): agentic-qe's own shipped default +// config.json violates its own writerIsNeverJuror invariant — defense: +// cognitum-low and jury: cognitum-high resolve to the same vendor per +// vendorOf() above, so a brand-new project fails validation before any user +// touches the file. Filed: proffesor-for-testing/agentic-qe#576. +export const UPSTREAM_JURY_VENDOR_ISSUE = 'https://github.com/proffesor-for-testing/agentic-qe/issues/576'; + +/** Compute a minimal fix for a writerIsNeverJuror violation: reassign `jury` + * to an already-configured provider whose vendor differs from every + * writer/defense vendor. Prefers `deeperReviewer`'s provider (already a + * second-look role) before falling back to the first other distinct-vendor + * seat. Returns null when the panel doesn't have the collision, or when no + * distinct-vendor seat exists to borrow from — this never invents a vendor + * the project hasn't already configured. Pure; touches nothing. */ +export function healJuryVendorCollision(routing) { + const panel = panelFromRouting(routing); + const writerVendors = new Set( + panel.filter((s) => s.role === 'defense' || s.role === 'writer').map((s) => vendorOf(s.provider)), + ); + const jury = panel.find((s) => s.role === 'jury'); + if (!jury || !writerVendors.has(vendorOf(jury.provider))) return null; + + const candidates = panel.filter((s) => s.role !== 'jury' && !writerVendors.has(vendorOf(s.provider))); + const preferred = candidates.find((s) => s.role === 'deeperReviewer') ?? candidates[0]; + if (!preferred) return null; + + return { role: 'jury', from: jury.provider, to: preferred.provider }; +} + export function qeCourtConfigPath(root) { return path.join(root, '.claude', 'skills', 'qe-court', 'config.json'); } diff --git a/src/lib/routing.mjs b/src/lib/routing.mjs index 4f5459e..0e6f047 100644 --- a/src/lib/routing.mjs +++ b/src/lib/routing.mjs @@ -40,14 +40,19 @@ export const SUBSCRIPTION_PROVIDERS = new Set(['claude-code', 'codex', 'ollama', // `--help`, and docs/PROVIDERS.md. NOT a hard allow-list: any model your host CLI // accepts also works — these are ak's curated picks. Web-verified on the date // below; model lines move fast, so re-check and let users override (ADR-0002/0003). +// Notes must state work-per-task, not only price: a note that compares cost +// WITHOUT saying "per-token" gets read as cost-per-task, which is the axis users +// actually pay on (a model needing 2-3x the agentic turns costs more per task at +// identical per-token price). Measured end-to-end in pacphi/retort versions-blog. export const MODEL_CATALOG_VERIFIED = '2026-07-24'; +export const COST_AXIS_NOTE = 'per-token price ≠ per-task cost — a model that needs more agentic turns costs more per task at the same per-token price'; export const MODEL_CATALOG = { claude: [ - { id: 'claude-opus-5', tier: 'reasoning', note: 'new top Opus — ~2× Opus 4.8 at the same price, near-Fable on coding/agentic; premium reasoning default' }, - { id: 'claude-sonnet-5', tier: 'balanced', note: 'near-Opus at lower cost — review, spec, release' }, + { id: 'claude-opus-5', tier: 'reasoning', note: 'new top Opus — same per-token price as 4.8, but ~2–3× the agentic turns on routine work; earns it at the hard end' }, + { id: 'claude-sonnet-5', tier: 'balanced', note: 'near-Opus capability at a lower per-token price — review, spec, release' }, { id: 'claude-fable-5', tier: 'flagship', note: 'top capability (Mythos-class, above Opus 5) — hardest problems' }, { id: 'claude-haiku-4-5-20251001', tier: 'fast', note: 'cheap/fast — high-volume mechanical work' }, - { id: 'claude-opus-4-8', tier: 'prior', note: 'prior Opus generation — same price as opus-5, kept for pinned configs' }, + { id: 'claude-opus-4-8', tier: 'prior', note: 'prior Opus generation — same per-token price, roughly half the turns on routine work' }, ], codex: [ { id: 'gpt-5.4', tier: 'flagship', note: 'coding + reasoning + agentic — recommended execution default' }, @@ -84,7 +89,10 @@ export function providerModelChoices(provider) { * host models (claude/codex) and provider-axis models (openrouter/GLM) reachable * via the aqe fallback chain. */ export function formatModelHelp() { - const lines = [`known-good models (verified ${MODEL_CATALOG_VERIFIED}; any model your host accepts also works):`]; + const lines = [ + `known-good models (verified ${MODEL_CATALOG_VERIFIED}; any model your host accepts also works):`, + ` ${COST_AXIS_NOTE}`, + ]; for (const host of HOSTS) { lines.push(` ${host}:`); for (const m of MODEL_CATALOG[host]) lines.push(` ${m.id.padEnd(28)} ${m.tier.padEnd(10)} ${m.note}`); @@ -224,6 +232,95 @@ export function seedDualRouting({ hosts = HOSTS, primary = DEFAULT_PRIMARY_HOST return policy; } +/** The catalog note for a model id (its cost-per-task characteristic), or null. + * Searches host + provider catalogs so a refresh diff can explain the trade. */ +export function modelNote(model) { + for (const list of [...Object.values(MODEL_CATALOG), ...Object.values(PROVIDER_MODEL_CATALOG)]) { + const m = list.find((e) => e.id === model); + if (m) return m.note; + } + return null; +} + +/** What seeding would produce for `act` TODAY, on the host the entry already + * sits on. A codex-primary seed mirrors every default route to the opposite + * host (swapRoute), so comparing against the raw default would read every + * mirrored route as diverged — mirror first when the hosts disagree. Pure. */ +function currentSeedFor(act, entry = {}) { + const def = DEFAULT_ROUTES[act]; + if (!def) return null; + return entry.host && entry.host !== def.host ? { ...def, ...swapRoute(def) } : def; +} + +/** + * Seeded routes whose model no longer matches what seeding would produce today — + * ak chose these values, the defaults have since moved, and nothing re-seeds them. + * Pure. + * + * Only `source === 'seeded'` entries are reported: a 'user' pin is deliberate + * intent and is never divergence. + * + * Which side is better is activity-dependent (a newer default can cost 2-3× the + * agentic turns on routine work), so callers must present this neutrally — a + * divergence to decide about, not a lag to clear. + */ +export function divergedRoutes(policy = {}) { + const out = []; + for (const act of ACTIVITIES) { + const p = policy[act]; + if (!p || p.source !== 'seeded') continue; + const want = currentSeedFor(act, p); + if (!want) continue; + const modelDiverged = !!(want.model && p.model && p.model !== want.model); + // An escalation rung is a routing decision too, and it diverges on its own + // schedule: a seed whose primary model never moved can still escalate into a + // model the defaults have since replaced. + const escalate = (p.escalate ?? []).map((rung, i) => { + const wantRung = want.escalate?.[i]; + return wantRung && rung.model !== wantRung.model + ? { host: rung.host, model: rung.model, defaultModel: wantRung.model } + : null; + }).filter(Boolean); + if (!modelDiverged && escalate.length === 0) continue; + out.push({ + activity: act, + host: p.host ?? want.host, + model: p.model, + defaultHost: want.host, + defaultModel: want.model, + modelDiverged, + escalate, + defaultNote: modelNote(want.model), + currentNote: modelNote(p.model), + }); + } + return out; +} + +/** + * Re-seed the named activities from the current defaults, returning a NEW policy. + * Only `source === 'seeded'` entries are eligible — a user pin survives untouched + * even when named. `activities` defaults to every diverged activity. Pure. + * @param {Record} [policy] + * @param {{ activities?: string[] }} [opts] + */ +export function refreshSeededRoutes(policy = {}, { activities } = {}) { + const eligible = new Set(divergedRoutes(policy).map((d) => d.activity)); + const want = new Set(activities ?? [...eligible]); + const out = { ...policy }; + for (const act of want) { + if (!eligible.has(act)) continue; + const seed = currentSeedFor(act, out[act]); + out[act] = { + host: seed.host, + model: seed.model, + ...(seed.escalate ? { escalate: seed.escalate } : {}), + source: 'seeded', + }; + } + return out; +} + /** * Projection #1 → aqe `agentOverrides`. Map curated QE agent-types to the * {provider, model} of their activity's effective route. Drops any whose provider diff --git a/src/lib/ruvector.mjs b/src/lib/ruvector.mjs new file mode 100644 index 0000000..27afc3d --- /dev/null +++ b/src/lib/ruvector.mjs @@ -0,0 +1,59 @@ +// RuVector CLI — drift detection for a global `ruvector` install ak does NOT own. +// +// Unlike ruflo/agentic-qe/the host CLIs, ruvector is not part of ak's managed set: +// users register it as an MCP server by hand, and a stale global then serves stale +// tools with nothing reporting it. So this is DETECTION + OPT-IN UPGRADE ONLY — +// presence is never nudged, and an absent ruvector produces no rows and no plan. +// Shaped after ruvnet-brain.mjs (present / classifyDrift / TTL-cached drift), but +// ruvector IS a plain npm global, so the version primitives come from versions.mjs +// rather than a parallel filesystem/GitHub-releases path. +import { installedVersion, latestVersion, cmpVersions } from './versions.mjs'; +import { loadKitConfig, saveKitConfig } from './config.mjs'; +import { ruvectorRegistered } from './mcp.mjs'; + +export const RUVECTOR_PKG = 'ruvector'; + +/** Installed globally? Detection only — ak never installs ruvector. */ +export function present() { + return !!installedVersion(RUVECTOR_PKG); +} + +/** May ak keep this global current? Registration IS the opt-in — a user who + * wired the MCP server up by hand has signalled they depend on the tool — and + * `kit.json ruvector:false` is the escape hatch (mirrors ruvnetBrain/aqe/agentdb). + * Both must hold: ak never touches a global nobody asked it to manage. */ +export function managed(cfg = loadKitConfig()) { + return cfg.ruvector !== false && ruvectorRegistered(); +} + +/** Pure drift classifier. `latest` null (offline / npm unreachable) is always + * "unknown", never "outdated" — mirrors ruvnet-brain's classifyDrift. */ +export function classifyDrift({ installed, latest }) { + if (!installed) return { present: false, outdated: false, installed: null, latest: latest ?? null }; + return { + present: true, + outdated: !!(latest && cmpVersions(latest, installed) > 0), + installed, + latest: latest ?? null, + }; +} + +/** Installed-vs-latest, TTL-cached in kit.json alongside the other version + * windows, so status/dashboard hit npm at most once per window. force=true + * bypasses the cache. Skips the network entirely when ruvector is absent — + * an unmanaged tool must not cost a probe. */ +export async function drift({ force = false } = {}) { + const installed = installedVersion(RUVECTOR_PKG); + if (!installed) return classifyDrift({ installed: null, latest: null }); + const cfg = loadKitConfig(); + const ttlMs = (cfg.versionCheck?.ttlHours ?? 24) * 3600_000; + const cached = cfg.versionCheck?.ruvector ?? {}; + const fresh = !force && cached.last && Date.now() - cached.last < ttlMs; + let latest = fresh ? cached.latest ?? null : null; + if (!fresh) { + latest = await latestVersion(RUVECTOR_PKG); + cfg.versionCheck = { ...cfg.versionCheck, ruvector: { last: Date.now(), latest } }; + try { saveKitConfig(cfg); } catch { /* read-only envs: next call re-fetches */ } + } + return classifyDrift({ installed, latest }); +} diff --git a/src/lib/versions.mjs b/src/lib/versions.mjs index f381748..50103ee 100644 --- a/src/lib/versions.mjs +++ b/src/lib/versions.mjs @@ -16,7 +16,7 @@ export function installedVersion(pkg) { } } -async function latestVersion(pkg, tag = 'latest') { +export async function latestVersion(pkg, tag = 'latest') { const r = await run('npm', ['view', `${pkg}@${tag}`, 'version'], { timeout: 20_000 }); return r.code === 0 ? r.stdout.trim() : null; } diff --git a/tests/kit/heal-qe-court.test.mjs b/tests/kit/heal-qe-court.test.mjs new file mode 100644 index 0000000..baecae4 --- /dev/null +++ b/tests/kit/heal-qe-court.test.mjs @@ -0,0 +1,106 @@ +// healQeCourtPanel — TEMPORARY heal for agentic-qe's own shipped default +// (proffesor-for-testing/agentic-qe#576: defense=cognitum-low + jury=cognitum-high +// collide on vendor, violating the config's own writerIsNeverJuror). Remove this +// test alongside heal.mjs's healQeCourtPanel once a released aqe fixes it upstream. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { healQeCourtPanel } from '../../src/lib/heal.mjs'; +import { qeCourtConfigPath } from '../../src/lib/qeCourt.mjs'; + +function tmpProject() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-heal-qecourt-')); + fs.mkdirSync(path.join(dir, '.git'), { recursive: true }); + return dir; +} +const rm = (dir) => fs.rmSync(dir, { recursive: true, force: true }); +function writeConfig(dir, routing, options = {}) { + const file = qeCourtConfigPath(dir); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ routing, options: { writerIsNeverJuror: true, ...options } })); + return file; +} + +test('healQeCourtPanel is a no-op when there is no project root', () => { + const r = healQeCourtPanel('/nonexistent-path-xyz'); + assert.equal(r.ok, true); + assert.match(r.detail, /no project root/); +}); + +test('healQeCourtPanel is a no-op when qe-court has not created its config yet', () => { + const dir = tmpProject(); + const r = healQeCourtPanel(dir); + assert.equal(r.ok, true); + assert.match(r.detail, /no qe-court config present/); + rm(dir); +}); + +test('healQeCourtPanel is a no-op (and touches nothing) when the panel is already valid', () => { + const dir = tmpProject(); + const file = writeConfig(dir, { + defense: { provider: 'cognitum-low' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'claude-code' }, + }); + const before = fs.readFileSync(file, 'utf8'); + const r = healQeCourtPanel(dir); + assert.equal(r.ok, true); + assert.match(r.detail, /already valid/); + assert.equal(fs.readFileSync(file, 'utf8'), before); + assert.equal(fs.existsSync(`${file}.bak`), false); + rm(dir); +}); + +test('healQeCourtPanel reassigns jury and backs up the original on a collision', () => { + const dir = tmpProject(); + const file = writeConfig(dir, { + defense: { provider: 'cognitum-low' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'cognitum-high' }, + deeperReviewer: { provider: 'codex' }, + }); + const r = healQeCourtPanel(dir); + assert.equal(r.ok, true); + assert.match(r.detail, /jury cognitum-high → codex/); + assert.match(r.detail, /agentic-qe\/issues\/576/); + + const after = JSON.parse(fs.readFileSync(file, 'utf8')); + assert.equal(after.routing.jury.provider, 'codex'); + assert.equal(after.routing.defense.provider, 'cognitum-low'); // untouched + assert.equal(after.routing.deeperReviewer.provider, 'codex'); // untouched + + const backup = JSON.parse(fs.readFileSync(`${file}.bak`, 'utf8')); + assert.equal(backup.routing.jury.provider, 'cognitum-high'); + rm(dir); +}); + +test('healQeCourtPanel is idempotent — running it twice makes no further change', () => { + const dir = tmpProject(); + const file = writeConfig(dir, { + defense: { provider: 'cognitum-low' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'cognitum-high' }, + }); + healQeCourtPanel(dir); + const afterFirst = fs.readFileSync(file, 'utf8'); + const r2 = healQeCourtPanel(dir); + assert.match(r2.detail, /already valid/); + assert.equal(fs.readFileSync(file, 'utf8'), afterFirst); + rm(dir); +}); + +test('healQeCourtPanel leaves an unfixable violation untouched', () => { + const dir = tmpProject(); + const file = writeConfig(dir, { + defense: { provider: 'cognitum-low' }, + 'prosecutor.sherlock': { provider: 'cognitum-high' }, + jury: { provider: 'cognitum-mid' }, // every seat is cognitum — no distinct vendor to borrow + }); + const before = fs.readFileSync(file, 'utf8'); + const r = healQeCourtPanel(dir); + assert.match(r.detail, /unfixable automatically/); + assert.equal(fs.readFileSync(file, 'utf8'), before); + rm(dir); +}); diff --git a/tests/kit/provider-credentials.test.mjs b/tests/kit/provider-credentials.test.mjs new file mode 100644 index 0000000..b699191 --- /dev/null +++ b/tests/kit/provider-credentials.test.mjs @@ -0,0 +1,355 @@ +// Issue #54 — aqe fallback-chain credential checking. +// +// Before this, three separate places validated a chain entry (applyAqeRouter, +// `x provider pick`, `ak status`) and none of them touched credentials: a chain +// whose second rung had no API key was written, reported `ok`, and only failed +// at QE-run time — far from the config that caused it. These tests pin the +// credential layer and the write-time warning that consumes it. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { loadKitConfig } from '../../src/lib/config.mjs'; +import { AQE_CONSTRUCTIBLE_PROVIDERS } from '../../src/lib/routing.mjs'; +import { + AQE_PROVIDER_TYPES, AQE_PROVIDER_CREDENTIALS, aqeProviderCredential, + credentialGaps, detectAqeProviders, fallbackSource, + applyAqeRouter, aqeRouterFile, +} from '../../src/lib/providers.mjs'; + +// An env with NO provider credentials at all — the baseline every "absent" case +// is measured against, so a key that happens to be exported on the developer's +// machine can never make a negative assertion pass vacuously. +const BARE_ENV = {}; + +const tmpProject = () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-cred-')); + fs.mkdirSync(path.join(dir, '.git'), { recursive: true }); + return dir; +}; +const rm = (dir) => fs.rmSync(dir, { recursive: true, force: true }); + +function defaultCfg() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-cred-cfg-')); + const cfg = loadKitConfig(path.join(dir, 'kit.json')); + rm(dir); + return cfg; +} + +/** Run `fn` with the named env vars removed from process.env, then restore. + * applyAqeRouter reads process.env (it has no env injection point), so the + * write-time tests must scrub the real env rather than pass a fixture. */ +function withoutEnv(keys, fn) { + const saved = {}; + for (const k of keys) { saved[k] = process.env[k]; delete process.env[k]; } + try { return fn(); } finally { + for (const k of keys) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + } +} + +// Every env var any descriptor can consult — scrubbing all of them makes the +// "keyless rung" fixture deterministic regardless of what this machine exports. +const ALL_CREDENTIAL_ENV = [...new Set( + Object.values(AQE_PROVIDER_CREDENTIALS).flatMap((d) => [...(d.keyEnv ?? []), ...(d.also ?? [])]), +)]; + +// ── the anti-divergence guard (the acceptance criterion) ──────────────────── + +test('every AQE_PROVIDER_TYPES member has a credential descriptor', () => { + // The original defect in miniature: AQE_PROVIDER_TYPES had ten members and the + // only credential probe was scoped to a FOUR-member list that was never + // reconciled with it, so six provider types were unverifiable by construction. + const missing = AQE_PROVIDER_TYPES.filter((p) => !(p in AQE_PROVIDER_CREDENTIALS)); + assert.deepEqual(missing, [], 'these aqe provider types are unverifiable by construction'); +}); + +test('no credential descriptor exists for a provider nothing can ever emit', () => { + // The other direction of the same guard, but deliberately NOT set-equality: + // `codex` legitimately has a descriptor while being absent from + // AQE_PROVIDER_TYPES (that list also gates AQE_LLM_PROVIDER validation, so + // widening it would be a separate behavior change) — policyToAgentOverrides + // emits `provider: 'codex'`, so those projected entries need one. The real + // invariant is that every descriptor corresponds to a provider some code path + // can actually produce, not that the two lists are identical. + const emittable = new Set([...AQE_PROVIDER_TYPES, ...AQE_CONSTRUCTIBLE_PROVIDERS]); + const orphans = Object.keys(AQE_PROVIDER_CREDENTIALS).filter((p) => !emittable.has(p)); + assert.deepEqual(orphans, [], 'a descriptor for a provider nothing emits is dead weight that will rot'); +}); + +test('codex — emitted by the agentOverrides projection — is credential-checkable', () => { + // Regression pin for the specific gap: a codex-hosted activity projects to + // `provider: 'codex'`, which a types-only credential map could not verify. + assert.ok(AQE_CONSTRUCTIBLE_PROVIDERS.includes('codex')); + const c = aqeProviderCredential('codex', { env: BARE_ENV }); + assert.equal(c.known, true, 'a projected codex override must be verifiable'); + assert.equal(c.billing, 'subscription', 'codex is host-credentialed, not metered'); +}); + +test('every credential descriptor declares exactly one credential mechanism', () => { + for (const [id, d] of Object.entries(AQE_PROVIDER_CREDENTIALS)) { + const mechanisms = [d.keyEnv ? 'keyEnv' : null, d.host ? 'host' : null, d.local ? 'local' : null].filter(Boolean); + assert.equal(mechanisms.length, 1, `${id} must declare exactly one of keyEnv/host/local, got ${mechanisms}`); + if (d.keyEnv) assert.ok(d.keyEnv.length > 0, `${id} keyEnv is non-empty`); + assert.ok(['metered', 'subscription', 'local'].includes(d.billing), `${id} billing is classified (got ${d.billing})`); + } +}); + +test('openrouter — the one credentialed provider on the reported machine — has a descriptor', () => { + // It was absent from API_PROVIDERS entirely, so the only live key on the box + // was invisible to every ak surface while keyless `openai` displayed as configured. + assert.ok(AQE_PROVIDER_CREDENTIALS.openrouter?.keyEnv?.includes('OPENROUTER_API_KEY')); +}); + +// ── aqeProviderCredential ─────────────────────────────────────────────────── + +test('aqeProviderCredential sees a key-present provider and names the var that satisfied it', () => { + const c = aqeProviderCredential('openrouter', { env: { OPENROUTER_API_KEY: 'sk-test' } }); + assert.equal(c.known, true); + assert.equal(c.present, true); + assert.equal(c.billing, 'metered'); + assert.equal(c.source, 'OPENROUTER_API_KEY'); + assert.deepEqual(c.missing, []); +}); + +test('aqeProviderCredential reports a keyless metered provider as absent, naming what would fix it', () => { + const c = aqeProviderCredential('openai', { env: BARE_ENV }); + assert.equal(c.present, false); + assert.ok(c.missing.length > 0, 'must say what is missing, not just that it is'); + assert.match(c.missing.join(' '), /OPENAI_API_KEY/); +}); + +test('aqeProviderCredential accepts any of a providers alternate key vars', () => { + for (const key of AQE_PROVIDER_CREDENTIALS.gemini.keyEnv) { + assert.equal(aqeProviderCredential('gemini', { env: { [key]: 'k' } }).present, true, `${key} satisfies gemini`); + } +}); + +test('aqeProviderCredential treats local providers as always usable and $0', () => { + for (const id of ['ollama', 'onnx']) { + const c = aqeProviderCredential(id, { env: BARE_ENV }); + assert.equal(c.present, true, `${id} needs no credential`); + assert.equal(c.billing, 'local'); + } +}); + +test('aqeProviderCredential routes claude-code to the host login, not an API key', () => { + // claude-code is credentialed by the frontier host's oauth/subscription — a + // keyEnv probe would report the working subscription rung as dead. + const c = aqeProviderCredential('claude-code', { env: BARE_ENV }); + assert.equal(c.known, true); + assert.equal(c.billing, 'subscription'); +}); + +test('aqeProviderCredential requires a providers extra hard-required vars, not just the key', () => { + // azure needs an endpoint; bedrock needs the secret half of the pair. A + // key-only check would call a half-configured provider live. + const azureKey = AQE_PROVIDER_CREDENTIALS['azure-openai'].keyEnv[0]; + assert.equal(aqeProviderCredential('azure-openai', { env: { [azureKey]: 'k' } }).present, false, + 'key without endpoint is not usable'); + const full = aqeProviderCredential('azure-openai', { env: { [azureKey]: 'k', AZURE_OPENAI_ENDPOINT: 'https://x' } }); + assert.equal(full.present, true); + + assert.equal(aqeProviderCredential('bedrock', { env: { AWS_ACCESS_KEY_ID: 'a' } }).present, false, + 'access key without secret is not usable'); +}); + +test('aqeProviderCredential reports an unknown provider as unknown, never as present', () => { + const c = aqeProviderCredential('not-a-provider', { env: BARE_ENV }); + assert.equal(c.known, false); + assert.equal(c.present, false, 'unknown must never read as usable'); +}); + +// ── credentialGaps (the shared basis for all three surfaces) ──────────────── + +test('credentialGaps is empty for a fully credentialed chain', () => { + const chain = [{ provider: 'openrouter' }, { provider: 'ollama' }]; + assert.deepEqual(credentialGaps(chain, { env: { OPENROUTER_API_KEY: 'k' } }), []); +}); + +test('credentialGaps names the dead rung of the exact reported chain', () => { + // The reproduction: claude-code + openai, no OPENAI_API_KEY. One live rung and + // a silent dead end underneath it. + const gaps = credentialGaps( + [{ provider: 'claude-code', models: ['claude-opus-5'] }, { provider: 'openai', models: ['gpt-5.6'] }], + { env: BARE_ENV }, + ); + assert.deepEqual(gaps.map((g) => g.provider), ['openai']); + assert.match(gaps[0].missing.join(' '), /OPENAI_API_KEY/); +}); + +test('credentialGaps preserves chain order so the report reads top-down', () => { + const gaps = credentialGaps( + [{ provider: 'openai' }, { provider: 'ollama' }, { provider: 'cognitum' }], + { env: BARE_ENV }, + ); + assert.deepEqual(gaps.map((g) => g.provider), ['openai', 'cognitum']); +}); + +test('credentialGaps ignores malformed entries instead of throwing', () => { + assert.deepEqual(credentialGaps([null, {}, { models: ['x'] }], { env: BARE_ENV }), []); + assert.deepEqual(credentialGaps([], { env: BARE_ENV }), []); +}); + +test('credentialGaps EXCLUDES an unknown provider — unverifiable is not uncredentialed', () => { + // Reporting "no credential for: mystery-provider" would be ak asserting a fact + // about a provider it has no descriptor for. Silence is the honest answer: we + // do not know, and saying we do would be the same over-claiming this whole + // issue exists to stop. + assert.deepEqual(credentialGaps([{ provider: 'mystery-provider' }], { env: BARE_ENV }), []); + // …and it must not swallow a real gap sitting next to the unknown one. + const mixed = credentialGaps( + [{ provider: 'mystery-provider' }, { provider: 'openai' }], + { env: BARE_ENV }, + ); + assert.deepEqual(mixed.map((g) => g.provider), ['openai']); +}); + +// ── hostAuth injection: BRANCH coverage only ─────────────────────────────── +// +// READ THIS BEFORE TRUSTING THE TESTS BELOW. +// +// `hostAuthState` carries a deliberate, NOT platform-gated inference: +// if (id === 'claude' && present) return { mode: 'oauth', ... } +// so `claude-code` resolves to `present: true` on every platform, with an empty +// env, always. Verified against the real function. +// +// That means these tests exercise the host-credential BRANCH by forcing a state +// `claude-code` cannot reach in production. They are worth keeping — the branch +// is real and shared with `codex` — but they must NOT be read as evidence that +// "ak warns when the Claude subscription is missing". Structurally, ak never +// does that for claude-code, by design. +// +// The genuinely reachable host-credential gap is `codex` (no such inference), +// and it is covered for real — no injection, real env + filesystem path — in +// status-viability.test.mjs, which sandboxes HOME. It cannot live here: this +// file does not redirect HOME, so a codex probe would read the developer's own +// ~/.codex/auth.json and pass or fail depending on whose machine ran it. + +test('the host-credential branch reports a logged-out host as a dead rung', () => { + const c = aqeProviderCredential('claude-code', { + env: BARE_ENV, + hostAuth: () => ({ mode: 'none', billing: 'unknown', source: null, note: null }), + }); + assert.equal(c.present, false); + assert.ok(c.missing.length > 0, 'it must say what would fix it'); + assert.match(c.missing.join(' '), /claude/i); +}); + +test('the host-credential branch reports a logged-in host as present and subscription-billed', () => { + const c = aqeProviderCredential('claude-code', { + env: BARE_ENV, + hostAuth: () => ({ mode: 'oauth', billing: 'subscription', source: 'login', note: null }), + }); + assert.equal(c.present, true); + assert.equal(c.billing, 'subscription'); + assert.deepEqual(c.missing, []); +}); + +test('claude-code is present WITHOUT injection — the inference this pins is unconditional', () => { + // Pins the production reality the two tests above deliberately bypass, so the + // gap between "branch tested" and "state reachable" can never be forgotten. + // If someone ever platform-gates that inference, this fails and the comment + // above stops being true — which is exactly when it needs re-reading. + const c = aqeProviderCredential('claude-code', { env: BARE_ENV }); + assert.equal(c.present, true, 'claude-code never reports a credential gap in production'); +}); + +test('credentialGaps propagates an injected host-auth failure across the whole chain', () => { + // Branch-level, same caveat as above. The property is that a dead host rung + // does not get special-cased into silence just because it is the primary. + const gaps = credentialGaps( + [{ provider: 'claude-code' }, { provider: 'openai' }], + { env: BARE_ENV, hostAuth: () => ({ mode: 'none', billing: 'unknown', source: null, note: null }) }, + ); + assert.deepEqual(gaps.map((g) => g.provider), ['claude-code', 'openai']); +}); + +// ── detectAqeProviders (what `ak x provider status` renders) ──────────────── + +test('detectAqeProviders covers every aqe provider type', () => { + const got = detectAqeProviders({ env: BARE_ENV }); + assert.deepEqual(Object.keys(got).sort(), [...AQE_PROVIDER_TYPES].sort()); +}); + +test('detectAqeProviders shows openrouter as present when OPENROUTER_API_KEY is set', () => { + // The explicit acceptance criterion — a credentialed provider must never be + // invisible while a keyless one displays as configured. + const got = detectAqeProviders({ env: { OPENROUTER_API_KEY: 'sk-test' } }); + assert.equal(got.openrouter.present, true); + assert.equal(got.openai.present, false, 'and a keyless provider must not read as configured'); +}); + +// ── fallbackSource (provenance stamping, #55 item 4) ─────────────────────── + +test('fallbackSource treats a legacy unstamped chain entry as a deliberate user pin', () => { + // Unrecoverable provenance must fail SAFE: never auto-touch what might have + // been typed by hand. + assert.equal(fallbackSource({ provider: 'openai', models: ['gpt-5.6'] }), 'user'); + assert.equal(fallbackSource(undefined), 'user'); +}); + +test('fallbackSource returns a stamped entrys own provenance', () => { + assert.equal(fallbackSource({ provider: 'openai', source: 'suggested' }), 'suggested'); + assert.equal(fallbackSource({ provider: 'openai', source: 'user' }), 'user'); +}); + +// ── applyAqeRouter: warn at write time, but STILL WRITE ──────────────────── + +test('applyAqeRouter warns about a keyless rung AND still writes the chain', () => { + // "Do not refuse to write" is explicit in the issue: the user may export the + // key later, and silently dropping a rung is worse than an inert one. + const dir = tmpProject(); + withoutEnv(ALL_CREDENTIAL_ENV, () => { + const cfg = defaultCfg(); + cfg.providers.aqeProvider = 'claude-code'; + cfg.providers.aqeFallback = [ + { provider: 'claude-code', models: ['claude-opus-5'] }, + { provider: 'openai', models: ['gpt-5.6'] }, + ]; + const res = applyAqeRouter(cfg, dir); + + assert.equal(res.changed, true, 'the config is still written'); + assert.match(res.detail, /no credential for/i, 'the detail names the credential gap'); + assert.match(res.detail, /openai/, 'and names the dead rung'); + assert.ok(!/claude-code — needs/.test(res.detail), 'the live subscription rung is not flagged'); + + const disk = JSON.parse(fs.readFileSync(aqeRouterFile(dir), 'utf8')); + assert.deepEqual(disk.fallbackChain.entries.map((e) => e.provider), ['claude-code', 'openai'], + 'the keyless rung is preserved on disk, not dropped'); + }); + rm(dir); +}); + +test('applyAqeRouter emits no credential warning when every rung is viable', () => { + const dir = tmpProject(); + withoutEnv(ALL_CREDENTIAL_ENV, () => { + process.env.OPENROUTER_API_KEY = 'sk-test'; + const cfg = defaultCfg(); + cfg.providers.aqeFallback = [ + { provider: 'openrouter', models: ['z-ai/glm-5.2'] }, + { provider: 'ollama', models: ['llama3'] }, + ]; + const res = applyAqeRouter(cfg, dir); + assert.equal(res.changed, true); + assert.ok(!/no credential for/i.test(res.detail), `unexpected credential warning: ${res.detail}`); + }); + rm(dir); +}); + +test('applyAqeRouter reports the models gap and the credential gap independently', () => { + // Two different defects on the same rung must both surface — an empty-models + // warning must not mask a missing key. + const dir = tmpProject(); + withoutEnv(ALL_CREDENTIAL_ENV, () => { + const cfg = defaultCfg(); + cfg.providers.aqeFallback = [{ provider: 'openai', models: [] }]; + const res = applyAqeRouter(cfg, dir); + assert.match(res.detail, /no models for/i); + assert.match(res.detail, /no credential for/i); + }); + rm(dir); +}); diff --git a/tests/kit/provider-refresh-cli.test.mjs b/tests/kit/provider-refresh-cli.test.mjs new file mode 100644 index 0000000..083e5bf --- /dev/null +++ b/tests/kit/provider-refresh-cli.test.mjs @@ -0,0 +1,220 @@ +// The user-facing halves of #54 and #55, exercised through a real CLI spawn. +// +// `ak x provider status` is read-only, and `refresh` only ever writes kit.json + +// the sandbox project's llm-config.json, so both are safe to run for real here +// (unlike `pick`, which can trigger installs). Everything is redirected at a +// throwaway HOME — see provider-cli.test.mjs for why all four env vars matter. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { AQE_PROVIDER_CREDENTIALS } from '../../src/lib/providers.mjs'; +import { DEFAULT_ROUTES } from '../../src/lib/routing.mjs'; + +const BIN = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../bin/agentic-kit.mjs'); + +// Cleared in every spawn so a key exported on the developer's machine can never +// decide the outcome of a credential assertion; individual tests add back +// exactly the one they are testing. +const ALL_CREDENTIAL_ENV = [...new Set( + Object.values(AQE_PROVIDER_CREDENTIALS).flatMap((d) => [...(d.keyEnv ?? []), ...(d.also ?? [])]), +)]; + +function sandbox(providers) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-refresh-home-')); + const cfgDir = path.join(home, '.config', 'agentic-kit'); + fs.mkdirSync(cfgDir, { recursive: true }); + fs.writeFileSync(path.join(cfgDir, 'kit.json'), JSON.stringify({ providers })); + const project = fs.mkdtempSync(path.join(os.tmpdir(), 'kit-refresh-proj-')); + fs.mkdirSync(path.join(project, '.git')); + return { home, project }; +} + +const rm = (...dirs) => { for (const d of dirs) fs.rmSync(d, { recursive: true, force: true }); }; + +const readKit = (home) => + JSON.parse(fs.readFileSync(path.join(home, '.config', 'agentic-kit', 'kit.json'), 'utf8')); + +function ak(args, { cwd, home, env = {} }) { + const cfgDir = path.join(home, '.config'); + const clean = { ...process.env }; + for (const k of ALL_CREDENTIAL_ENV) delete clean[k]; + const r = spawnSync(process.execPath, [BIN, ...args], { + encoding: 'utf8', + cwd, + env: { ...clean, NO_COLOR: '1', HOME: home, USERPROFILE: home, XDG_CONFIG_HOME: cfgDir, APPDATA: cfgDir, ...env }, + }); + return { ...r, all: `${r.stdout}${r.stderr}` }; +} + +const DUAL = { claude: true, codex: true }; +const FORBIDDEN_FRAMING = /\b(stale|outdated|superseded)\b/i; + +/** architecture + design pinned to the prior Opus generation (seeded), plus a + * deliberate user pin that must survive everything. */ +const divergedProviders = () => ({ + hosts: { ...DUAL }, + dualRouting: { + architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + design: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + debugging: { host: 'claude', model: 'claude-opus-4-8', source: 'user' }, + }, +}); + +// ── #54: `ak x provider status` credential visibility ────────────────────── + +test('x provider status shows openrouter as credentialed when OPENROUTER_API_KEY is set', () => { + // The reported machine's one live key was invisible to every ak surface, + // while keyless openai displayed as a configured fallback. + const { home, project } = sandbox({ hosts: { claude: true } }); + const r = ak(['x', 'provider', 'status'], { cwd: project, home, env: { OPENROUTER_API_KEY: 'sk-test' } }); + const line = r.all.split('\n').find((l) => /^\s*openrouter\b/.test(l)); + assert.ok(line, `openrouter must appear in the provider table:\n${r.all}`); + assert.match(line, /key present/, `openrouter is credentialed here, got: ${line}`); + rm(home, project); +}); + +test('x provider status shows a keyless provider as having no key, naming the var', () => { + const { home, project } = sandbox({ hosts: { claude: true } }); + const r = ak(['x', 'provider', 'status'], { cwd: project, home }); + const line = r.all.split('\n').find((l) => /^\s*openai\b/.test(l)); + assert.ok(line, `openai must appear in the provider table:\n${r.all}`); + assert.match(line, /no key/); + assert.match(line, /OPENAI_API_KEY/); + rm(home, project); +}); + +test('x provider status warns that a keyless chain rung fails over into nothing', () => { + // The exact reproduction: claude-code + openai, no OPENAI_API_KEY. + const { home, project } = sandbox({ + hosts: { ...DUAL }, + aqeFallback: [ + { provider: 'claude-code', models: ['claude-opus-5'] }, + { provider: 'openai', models: ['gpt-5.6'] }, + ], + }); + const r = ak(['x', 'provider', 'status'], { cwd: project, home }); + assert.match(r.all, /openai.*no credential|no credential.*openai/is); + assert.match(r.all, /fail over into nothing/i, 'the consequence must be stated, not just the gap'); + rm(home, project); +}); + +test('x provider status emits no credential warning when every rung is viable', () => { + const { home, project } = sandbox({ + hosts: { ...DUAL }, + aqeFallback: [{ provider: 'openrouter', models: ['z-ai/glm-5.2'] }], + }); + const r = ak(['x', 'provider', 'status'], { cwd: project, home, env: { OPENROUTER_API_KEY: 'sk-test' } }); + assert.ok(!/fail over into nothing/i.test(r.all), `unexpected warning:\n${r.all}`); + rm(home, project); +}); + +// ── #55: divergence shown neutrally in the routing table ────────────────── + +test('x provider status marks a diverged seeded route against its current default', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'status'], { cwd: project, home }); + const line = r.all.split('\n').find((l) => /^\s*architecture\b/.test(l)); + assert.ok(line, `routing table must list architecture:\n${r.all}`); + assert.match(line, /diverges from default/i); + assert.match(line, new RegExp(DEFAULT_ROUTES.architecture.model), 'the current default is shown inline'); + rm(home, project); +}); + +test('x provider status never frames a diverged route as stale, outdated, or superseded', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'status'], { cwd: project, home }); + for (const line of r.all.split('\n').filter((l) => /diverge/i.test(l))) { + assert.ok(!FORBIDDEN_FRAMING.test(line), `neutral framing violated: ${line}`); + } + rm(home, project); +}); + +test('x provider status does not mark a user-pinned older model as diverged', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'status'], { cwd: project, home }); + const line = r.all.split('\n').find((l) => /^\s*debugging\b/.test(l)); + assert.ok(line, `routing table must list debugging:\n${r.all}`); + assert.ok(!/diverges from default/i.test(line), `a deliberate pin is not drift: ${line}`); + rm(home, project); +}); + +// ── #55: `ak x provider refresh` ────────────────────────────────────────── + +test('x provider refresh re-seeds only the activities named with --activity', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'refresh', '--activity', 'architecture'], { cwd: project, home }); + assert.equal(r.status, 0, r.all); + const routing = readKit(home).providers.dualRouting; + assert.equal(routing.architecture.model, DEFAULT_ROUTES.architecture.model, 'named route refreshed'); + assert.equal(routing.design.model, 'claude-opus-4-8', 'unnamed route deliberately left as it was'); + rm(home, project); +}); + +test('x provider refresh leaves a user-pinned route untouched even when named', () => { + const { home, project } = sandbox(divergedProviders()); + ak(['x', 'provider', 'refresh', '--activity', 'debugging'], { cwd: project, home }); + const routing = readKit(home).providers.dualRouting; + assert.equal(routing.debugging.model, 'claude-opus-4-8'); + assert.equal(routing.debugging.source, 'user'); + rm(home, project); +}); + +test('x provider refresh --yes re-seeds every diverged route and converges', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'refresh', '--yes'], { cwd: project, home }); + assert.equal(r.status, 0, r.all); + const routing = readKit(home).providers.dualRouting; + assert.equal(routing.architecture.model, DEFAULT_ROUTES.architecture.model); + assert.equal(routing.design.model, DEFAULT_ROUTES.design.model); + assert.equal(routing.debugging.model, 'claude-opus-4-8', 'user pins survive an all-refresh'); + + const second = ak(['x', 'provider', 'refresh'], { cwd: project, home }); + assert.match(second.all, /no seeded routes diverge/i, 'refreshing converges — the second run is a no-op'); + rm(home, project); +}); + +test('x provider refresh prints the cost-per-task trade for BOTH models, not just ids', () => { + // The whole point of the neutral framing: the user is being handed a decision + // that genuinely goes both ways, so both sides need their characteristic. + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'refresh', '--activity', 'architecture'], { cwd: project, home }); + assert.match(r.all, /turns/i, 'the work-per-task axis must appear, not only price'); + assert.match(r.all, /claude-opus-4-8/); + assert.match(r.all, new RegExp(DEFAULT_ROUTES.architecture.model)); + rm(home, project); +}); + +test('x provider refresh is a no-op on a policy seeded from current defaults', () => { + const { home, project } = sandbox({ + hosts: { ...DUAL }, + dualRouting: { + architecture: { host: 'claude', model: DEFAULT_ROUTES.architecture.model, source: 'seeded' }, + }, + }); + const r = ak(['x', 'provider', 'refresh', '--yes'], { cwd: project, home }); + assert.equal(r.status, 0); + assert.match(r.all, /no seeded routes diverge/i); + rm(home, project); +}); + +test('x provider refresh ignores an unknown activity instead of failing', () => { + const { home, project } = sandbox(divergedProviders()); + const r = ak(['x', 'provider', 'refresh', '--activity', 'not-an-activity'], { cwd: project, home }); + assert.equal(r.status, 0, r.all); + assert.match(r.all, /unknown activity/i); + assert.equal(readKit(home).providers.dualRouting.architecture.model, 'claude-opus-4-8', 'nothing refreshed'); + rm(home, project); +}); + +test('refresh is advertised as a subcommand and rejects an unknown one', () => { + const { home, project } = sandbox({ hosts: { claude: true } }); + const help = ak(['x', 'provider', '--help'], { cwd: project, home }); + assert.match(help.all, /refresh/, 'the opt-in path must be discoverable'); + const bad = ak(['x', 'provider', 'bogus'], { cwd: project, home }); + assert.match(bad.all, /status\|pick\|refresh\|off/); + rm(home, project); +}); diff --git a/tests/kit/qeCourt.test.mjs b/tests/kit/qeCourt.test.mjs index 094846a..08211d0 100644 --- a/tests/kit/qeCourt.test.mjs +++ b/tests/kit/qeCourt.test.mjs @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import { vendorOf, panelFromRouting, validatePanel, qeCourtConfigPath, readQeCourtConfig, + healJuryVendorCollision, UPSTREAM_JURY_VENDOR_ISSUE, } from '../../src/lib/qeCourt.mjs'; // vendorOf — ported from qe-court's referee.js @@ -106,3 +107,71 @@ test('readQeCourtConfig reads an existing config.json', () => { assert.equal(cfg.routing.jury.provider, 'cognitum-high'); fs.rmSync(dir, { recursive: true, force: true }); }); + +// healJuryVendorCollision — temporary heal for agentic-qe's own shipped +// default (proffesor-for-testing/agentic-qe#576); remove once fixed upstream. + +test('healJuryVendorCollision reassigns jury to deeperReviewer\'s vendor when it collides with the writer', () => { + const routing = { + defense: { provider: 'cognitum-low' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'cognitum-high' }, + deeperReviewer: { provider: 'codex' }, + }; + const fix = healJuryVendorCollision(routing); + assert.deepEqual(fix, { role: 'jury', from: 'cognitum-high', to: 'codex' }); +}); + +test('healJuryVendorCollision matches agentic-qe\'s actual shipped default', () => { + // exact routing block agentic-qe@3.13.2 ships in assets/skills/qe-court/config.json + const routing = { + defense: { provider: 'cognitum-low' }, + 'prosecutor.devils-advocate': { provider: 'cognitum-mid' }, + 'prosecutor.brutal-honesty': { provider: 'claude-code', model: 'sonnet' }, + 'prosecutor.sherlock': { provider: 'cognitum-high' }, + 'prosecutor.security-scanner': { provider: 'cognitum-mid' }, + 'prosecutor.mutation': { provider: 'ollama' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'cognitum-high' }, + deeperReviewer: { provider: 'codex' }, + }; + assert.deepEqual(validatePanel(panelFromRouting(routing)), ['writerIsNeverJuror']); + assert.deepEqual(healJuryVendorCollision(routing), { role: 'jury', from: 'cognitum-high', to: 'codex' }); +}); + +test('healJuryVendorCollision falls back to any other distinct-vendor seat when deeperReviewer is absent', () => { + const routing = { + defense: { provider: 'cognitum-low' }, + 'prosecutor.brutal-honesty': { provider: 'claude-code' }, + jury: { provider: 'cognitum-high' }, + }; + const fix = healJuryVendorCollision(routing); + assert.deepEqual(fix, { role: 'jury', from: 'cognitum-high', to: 'claude-code' }); +}); + +test('healJuryVendorCollision returns null when the panel is already valid', () => { + const routing = { + defense: { provider: 'cognitum-low' }, + 'prosecutor.codex-review': { provider: 'codex' }, + jury: { provider: 'claude-code' }, + }; + assert.equal(healJuryVendorCollision(routing), null); +}); + +test('healJuryVendorCollision returns null when there is no distinct-vendor seat to borrow from', () => { + const routing = { + defense: { provider: 'cognitum-low' }, + 'prosecutor.sherlock': { provider: 'cognitum-high' }, + jury: { provider: 'cognitum-mid' }, + }; + assert.equal(healJuryVendorCollision(routing), null); +}); + +test('healJuryVendorCollision returns null when there is no jury seat at all', () => { + const routing = { defense: { provider: 'cognitum-low' }, 'prosecutor.mutation': { provider: 'ollama' } }; + assert.equal(healJuryVendorCollision(routing), null); +}); + +test('UPSTREAM_JURY_VENDOR_ISSUE points at the filed agentic-qe issue', () => { + assert.equal(UPSTREAM_JURY_VENDOR_ISSUE, 'https://github.com/proffesor-for-testing/agentic-qe/issues/576'); +}); diff --git a/tests/kit/routing-divergence.test.mjs b/tests/kit/routing-divergence.test.mjs new file mode 100644 index 0000000..6b2d86c --- /dev/null +++ b/tests/kit/routing-divergence.test.mjs @@ -0,0 +1,372 @@ +// Issue #55 — seeded-routing divergence detection, and the catalog-note wording +// that the divergence UX depends on. +// +// The defect #55 describes is INVISIBILITY, not the older model: a machine +// seeded before a catalog bump keeps routing to the prior model and no ak +// surface says so. Measured end-to-end (retort versions-blog.md) the diverged +// pin is *cheaper and faster* on routine work, so every assertion here pins +// NEUTRAL framing — `info` not `warn`, "diverges" not "stale". A test that let +// the severity drift to `warn` would push users to spend 2-3.4x the agentic +// turns to clear a lint. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + MODEL_CATALOG, PROVIDER_MODEL_CATALOG, formatModelHelp, + DEFAULT_ROUTES, ACTIVITIES, seedDualRouting, divergedRoutes, refreshSeededRoutes, + resolveRoutes, COST_AXIS_NOTE, +} from '../../src/lib/routing.mjs'; + +const PKG_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); + +// ── item 3 (standalone): catalog notes must not conflate price with cost ───── +// Per-TOKEN price is at parity between opus-5 and opus-4-8; per-TASK cost is +// not (opus-5 takes 2-3.4x the agentic turns, and cache reads scale ~turns^2). +// The notes are rendered into `pick --help`, the interactive prompt, and +// docs/PROVIDERS.md — anyone choosing a model reads them as cost-per-task. +const COST_CLAIM = /(same|lower|higher)\s+(price|cost)/i; +const PER_TOKEN_QUALIFIER = /per[-\s]token/i; + +const allCatalogEntries = () => [ + ...Object.entries(MODEL_CATALOG).flatMap(([host, ms]) => ms.map((m) => [`${host}/${m.id}`, m])), + ...Object.entries(PROVIDER_MODEL_CATALOG).flatMap(([prov, ms]) => ms.map((m) => [`${prov}/${m.id}`, m])), +]; + +test('no MODEL_CATALOG note states a price/cost comparison without qualifying it as per-token', () => { + const offenders = []; + for (const [label, m] of allCatalogEntries()) { + const note = m.note ?? ''; + if (COST_CLAIM.test(note) && !PER_TOKEN_QUALIFIER.test(note)) offenders.push(`${label}: ${note}`); + } + assert.deepEqual(offenders, [], + 'a bare "same/lower/higher price|cost" claim reads as cost-per-task; qualify it with "per-token"'); +}); + +test('every catalog entry carries a non-empty note (the cost-per-task vehicle)', () => { + for (const [label, m] of allCatalogEntries()) { + assert.equal(typeof m.note, 'string', `${label} has a note`); + assert.ok(m.note.trim().length > 0, `${label} note is non-empty`); + } +}); + +test('formatModelHelp headers that per-token price is not per-task cost', () => { + const help = formatModelHelp(); + assert.match(help, PER_TOKEN_QUALIFIER, 'help must name the per-token axis explicitly'); + assert.match(help, /per[-\s]task/i, 'help must name the per-task axis it is contrasted with'); +}); + +// ── divergedRoutes: pure detection ────────────────────────────────────────── + +test('divergedRoutes returns [] for a policy seeded from the CURRENT defaults', () => { + assert.deepEqual(divergedRoutes(seedDualRouting({ hosts: ['claude', 'codex'] })), []); +}); + +test('divergedRoutes returns [] for an empty policy (nothing persisted, nothing to diverge)', () => { + assert.deepEqual(divergedRoutes({}), []); + assert.deepEqual(divergedRoutes(), []); +}); + +test('divergedRoutes reports a seeded entry whose model no longer matches the default', () => { + // The exact #55 reproduction: a machine seeded pre-alpha.22 still pinned to + // the prior Opus generation while DEFAULT_ROUTES moved to claude-opus-5. + const policy = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' } }; + const out = divergedRoutes(policy); + assert.equal(out.length, 1); + assert.equal(out[0].activity, 'architecture'); + assert.equal(out[0].model, 'claude-opus-4-8', 'reports the persisted (current) model'); + assert.equal(out[0].defaultModel, DEFAULT_ROUTES.architecture.model, 'and the default it diverges from'); +}); + +test('divergedRoutes NEVER reports a source:user entry, even pinned to an older model', () => { + // A deliberate pin is intent, not drift. Reporting it would nag the user + // about a choice they made on purpose. + const policy = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'user' } }; + assert.deepEqual(divergedRoutes(policy), []); +}); + +test('divergedRoutes treats an unstamped persisted entry as a user pin (never reported)', () => { + // resolveRoutes already defaults a source-less entry to 'user' (a hand edit is + // intent). Divergence detection must inherit that, or a legacy hand-edited + // kit.json gets flagged as machine drift. + assert.deepEqual(divergedRoutes({ architecture: { host: 'claude', model: 'claude-opus-4-8' } }), []); +}); + +test('the unstamped→user rule is pinned on BOTH mechanisms that implement it', () => { + // "unstamped is never reported" is enforced twice, independently: resolveRoutes + // defaults `source` to 'user' (routing.mjs), and divergedRoutes reads + // policy[act].source directly and skips anything !== 'seeded'. Asserting only + // the divergedRoutes half would let a future change to resolveRoutes' defaulting + // silently split the two — this pins the equivalence itself. + const legacy = { architecture: { host: 'claude', model: 'claude-opus-4-8' } }; // no `source` + assert.equal(resolveRoutes(legacy).architecture.source, 'user'); + assert.deepEqual(divergedRoutes(legacy), []); +}); + +test('the codex-primary clean-seed assertion actually covers a LADDER-bearing activity', () => { + // Guards a bug one level down from the top-level mirroring one: if the default + // were mirrored via a top-level host/model swap while `def.escalate` were left + // unmirrored, a codex-primary seed would pass the top-level check and then + // false-positive on every rung. `[]` over a ladder-less sample would not catch + // that — so first prove ladders are present, THEN prove they are clean. + const seed = seedDualRouting({ primary: 'codex' }); + const withLadders = Object.entries(seed).filter(([, r]) => r.escalate?.length); + assert.ok(withLadders.length > 0, + 'anti-vacuity: a codex-primary seed must carry ladders, else this proves nothing'); + for (const [, r] of withLadders) { + for (const rung of r.escalate) { + assert.equal(rung.host, 'codex', 'the LADDER must be mirrored too, not just the top-level route'); + } + } + assert.deepEqual(divergedRoutes(seed), []); +}); + +test('a corrupted rung on a MIRRORED activity is still reported (mirroring is not amnesty)', () => { + const seed = seedDualRouting({ primary: 'codex' }); + const [act] = Object.entries(seed).find(([, r]) => r.escalate?.length); + seed[act].escalate[0].model = 'gpt-5-codex-mini'; + const [d] = divergedRoutes(seed); + assert.equal(d.activity, act); + assert.equal(d.modelDiverged, false, 'only the rung moved — the primary model is untouched'); + assert.equal(d.escalate[0].model, 'gpt-5-codex-mini'); + assert.equal(d.escalate[0].defaultModel, 'gpt-5.4', 'compared against the MIRRORED default rung'); +}); + +test('divergedRoutes ignores a seeded entry that still matches the default', () => { + const policy = { + architecture: { host: 'claude', model: DEFAULT_ROUTES.architecture.model, source: 'seeded' }, + design: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + }; + assert.deepEqual(divergedRoutes(policy).map((d) => d.activity), ['design']); +}); + +/** The pre-alpha.22 seed: a REAL seeded policy (escalation ladders included, + * exactly as seedDualRouting writes them) with every claude-opus-5 pin rewound + * to claude-opus-4-8 — primary models and escalation rungs alike. Building this + * from seedDualRouting rather than by hand matters: a fixture that silently + * dropped `escalate` would under-report divergence and hide the escalate-only + * case entirely. */ +function priorCatalogSeed({ primary = 'claude' } = {}) { + const rewind = (m) => (m === 'claude-opus-5' ? 'claude-opus-4-8' : m); + const policy = seedDualRouting({ hosts: ['claude', 'codex'], primary }); + const out = {}; + for (const [act, r] of Object.entries(policy)) { + out[act] = { + ...r, + model: rewind(r.model), + ...(r.escalate ? { escalate: r.escalate.map((e) => ({ ...e, model: rewind(e.model) })) } : {}), + }; + } + return out; +} + +test('divergedRoutes reports the full #55 reproduction: SIX routes, escalation rungs included', () => { + // The issue observed exactly six on a machine tracking main. Four diverge on + // their primary model; implementation/testing still pin gpt-5.4 (unchanged) + // but escalate into the prior Opus — a routing decision that diverged on its + // own schedule and would be missed by a primary-model-only comparison. + const diverged = divergedRoutes(priorCatalogSeed()); + assert.deepEqual(diverged.map((d) => d.activity).sort(), + ['architecture', 'debugging', 'design', 'implementation', 'security-analysis', 'testing']); + assert.equal(diverged.length, 6); +}); + +test('an escalate-only divergence is flagged as such, never as a bogus same-model diff', () => { + // implementation's primary model did not move, so rendering it as + // "gpt-5.4 vs gpt-5.4" would be nonsense. `modelDiverged` lets callers branch. + const d = divergedRoutes(priorCatalogSeed()).find((x) => x.activity === 'implementation'); + assert.equal(d.modelDiverged, false, 'the primary model is unchanged'); + assert.equal(d.model, d.defaultModel, 'and both sides agree on it'); + assert.ok(d.escalate.length > 0, 'the divergence lives entirely in the escalation ladder'); + assert.equal(d.escalate[0].model, 'claude-opus-4-8'); + assert.equal(d.escalate[0].defaultModel, DEFAULT_ROUTES.implementation.escalate[0].model); +}); + +test('a primary-model divergence carries modelDiverged and an empty escalation delta', () => { + const d = divergedRoutes(priorCatalogSeed()).find((x) => x.activity === 'architecture'); + assert.equal(d.modelDiverged, true); + assert.deepEqual(d.escalate, [], 'architecture has no ladder to diverge'); +}); + +// ── swapRoute mirroring, attacked directly ───────────────────────────────── +// `divergedRoutes` mirrors the default via swapRoute when an entry's host +// differs from the default's. Get this wrong in EITHER direction and the +// feature is worthless: too eager and every codex-primary route false-positives +// (noise worse than the original invisibility); too lax and mirroring becomes a +// blanket amnesty that hides real drift. Both directions are attacked here. + +test('divergedRoutes returns [] for a CODEX-PRIMARY seed (mirrored routes are not drift)', () => { + assert.deepEqual(divergedRoutes(seedDualRouting({ hosts: ['claude', 'codex'], primary: 'codex' })), []); +}); + +test('a codex-primary seed really is host-mirrored — the fixture is not vacuous', () => { + // Guards the test above from passing for the wrong reason. If swapRoute ever + // became a no-op, the codex-primary seed would equal the claude-primary one + // and "[] means mirroring works" would prove nothing at all. + const claudePrimary = seedDualRouting({ hosts: ['claude', 'codex'], primary: 'claude' }); + const codexPrimary = seedDualRouting({ hosts: ['claude', 'codex'], primary: 'codex' }); + const flipped = ACTIVITIES.filter((a) => claudePrimary[a] && codexPrimary[a] + && claudePrimary[a].host !== codexPrimary[a].host); + assert.ok(flipped.length > 0, 'a codex-primary seed must actually mirror hosts, or this suite is vacuous'); +}); + +test('mirroring is scoped to the host swap — it does not excuse a wrong model on the mirrored host', () => { + // The dangerous failure: treating "host differs from default" as licence to + // skip the model comparison entirely. Here the host legitimately differs + // (codex-primary mirror) AND the model is wrong; it must still be reported. + const seed = seedDualRouting({ hosts: ['claude', 'codex'], primary: 'codex' }); + const act = ACTIVITIES.find((a) => seed[a] && seed[a].host !== DEFAULT_ROUTES[a].host); + assert.ok(act, 'need at least one mirrored activity to attack'); + const policy = { ...seed, [act]: { ...seed[act], model: 'definitely-not-the-seeded-model' } }; + const hit = divergedRoutes(policy).find((d) => d.activity === act); + assert.ok(hit, `a mirrored route with a wrong model must still be reported (${act})`); + assert.equal(hit.modelDiverged, true); + assert.equal(hit.defaultModel, seed[act].model, 'compared against the MIRRORED default, not the raw one'); +}); + +test('a claude-primary seed is not silently excused by the mirroring branch either', () => { + // Same attack from the unmirrored side: hosts match the default, so the + // mirroring branch must not engage at all. + const seed = seedDualRouting({ hosts: ['claude', 'codex'], primary: 'claude' }); + const policy = { ...seed, architecture: { ...seed.architecture, model: 'definitely-not-the-seeded-model' } }; + const hit = divergedRoutes(policy).find((d) => d.activity === 'architecture'); + assert.ok(hit); + assert.equal(hit.defaultModel, DEFAULT_ROUTES.architecture.model); +}); + +test('a hand-moved host with a current model is not reported as divergence', () => { + // A user (or a primary swap) can put an activity on the other host while its + // model is exactly what seeding would produce there. That is not drift. + const mirrored = seedDualRouting({ hosts: ['claude', 'codex'], primary: 'codex' }); + const act = ACTIVITIES.find((a) => mirrored[a] && mirrored[a].host !== DEFAULT_ROUTES[a].host); + assert.deepEqual(divergedRoutes({ [act]: { ...mirrored[act] } }), []); +}); + +test('a codex-primary seed on the PRIOR catalog still reports real divergence', () => { + // The mirroring must not suppress genuine drift — only false positives. + const diverged = divergedRoutes(priorCatalogSeed({ primary: 'codex' })); + assert.ok(diverged.length > 0, 'a mirrored seed pinned to the prior catalog still diverges'); + for (const d of diverged) { + assert.ok(d.modelDiverged || d.escalate.length > 0, `${d.activity} reported with no actual delta`); + } +}); + +test('refreshSeededRoutes clears an escalate-only divergence too', () => { + const policy = priorCatalogSeed(); + const out = refreshSeededRoutes(policy, { activities: ['implementation'] }); + assert.deepEqual(divergedRoutes(out).map((d) => d.activity).sort(), + ['architecture', 'debugging', 'design', 'security-analysis', 'testing'], + 'implementation converged, including its ladder'); + assert.equal(out.implementation.escalate[0].model, DEFAULT_ROUTES.implementation.escalate[0].model); +}); + +test('refreshing the full prior-catalog seed converges every one of the six', () => { + assert.deepEqual(divergedRoutes(refreshSeededRoutes(priorCatalogSeed())), []); +}); + +test('divergedRoutes is pure — it does not mutate the policy it is given', () => { + const policy = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' } }; + const before = JSON.stringify(policy); + divergedRoutes(policy); + assert.equal(JSON.stringify(policy), before); +}); + +test('each diverged entry carries BOTH models cost-per-task characteristics, not just IDs', () => { + // The refresh diff has to let the user decide, and the decision genuinely goes + // both ways — so the note (the work-per-task vehicle) must ride along for the + // current pin AND the default it would move to. + const [d] = divergedRoutes({ architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' } }); + for (const key of ['currentNote', 'defaultNote']) { + assert.equal(typeof d[key], 'string', `${key} present`); + assert.ok(d[key].trim().length > 0, `${key} non-empty`); + } + assert.notEqual(d.currentNote, d.defaultNote, 'the two sides must be distinguishable'); +}); + +// ── refreshSeededRoutes: the opt-in, per-activity correction ──────────────── + +test('refreshSeededRoutes re-seeds every diverged activity when none are named', () => { + const policy = { + architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + design: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + }; + const out = refreshSeededRoutes(policy); + assert.equal(out.architecture.model, DEFAULT_ROUTES.architecture.model); + assert.equal(out.design.model, DEFAULT_ROUTES.design.model); + assert.deepEqual(divergedRoutes(out), [], 'refreshing converges — nothing diverges afterwards'); +}); + +test('refreshSeededRoutes leaves a source:user pin untouched even when explicitly named', () => { + // The acceptance criterion that matters most: a deliberate pin must survive a + // refresh the user aimed straight at it. + const policy = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'user' } }; + const out = refreshSeededRoutes(policy, { activities: ['architecture'] }); + assert.deepEqual(out.architecture, policy.architecture); + assert.equal(out.architecture.source, 'user'); +}); + +test('refreshSeededRoutes is per-activity selectable — unnamed diverged routes stay put', () => { + // The right answer differs across the six, so refresh must not be all-or-nothing. + const policy = { + architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + design: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' }, + }; + const out = refreshSeededRoutes(policy, { activities: ['architecture'] }); + assert.equal(out.architecture.model, DEFAULT_ROUTES.architecture.model, 'named route refreshed'); + assert.equal(out.design.model, 'claude-opus-4-8', 'unnamed route deliberately left diverged'); +}); + +test('refreshSeededRoutes keeps the refreshed entry stamped seeded (so it stays refreshable)', () => { + const out = refreshSeededRoutes({ architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' } }); + assert.equal(out.architecture.source, 'seeded'); +}); + +test('refreshSeededRoutes is pure — the input policy is not mutated', () => { + const policy = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'seeded' } }; + const before = JSON.stringify(policy); + const out = refreshSeededRoutes(policy); + assert.equal(JSON.stringify(policy), before, 'input untouched'); + assert.notEqual(out, policy, 'a NEW policy is returned'); +}); + +test('refreshSeededRoutes is a no-op on a policy seeded from current defaults', () => { + const policy = seedDualRouting({ hosts: ['claude', 'codex'] }); + assert.deepEqual(refreshSeededRoutes(policy), policy); +}); + +test('COST_AXIS_NOTE contrasts the per-token and per-task axes by name', () => { + assert.match(COST_AXIS_NOTE, PER_TOKEN_QUALIFIER); + assert.match(COST_AXIS_NOTE, /per[-\s]task/i); +}); + +// ── docs/PROVIDERS.md renders the catalog — keep the two from drifting ────── + +test('docs/PROVIDERS.md renders every HOST model note verbatim', () => { + // The notes reach users through three surfaces (pick --help, the interactive + // prompt, and this table). A doc regenerated by hand drifts silently, which is + // how the misleading price wording survived a catalog bump in the first place. + // Scoped to MODEL_CATALOG: the provider-axis models are documented in prose + // rather than the generated table (asserted separately below). + const doc = fs.readFileSync(path.join(PKG_ROOT, 'docs', 'PROVIDERS.md'), 'utf8'); + for (const [host, models] of Object.entries(MODEL_CATALOG)) { + for (const m of models) { + assert.ok(doc.includes(m.note), `docs/PROVIDERS.md is stale for ${host}/${m.id} — missing note: ${m.note}`); + } + } +}); + +test('docs/PROVIDERS.md documents every provider-axis (aqe-fallback) model by id', () => { + const doc = fs.readFileSync(path.join(PKG_ROOT, 'docs', 'PROVIDERS.md'), 'utf8'); + for (const [prov, models] of Object.entries(PROVIDER_MODEL_CATALOG)) { + for (const m of models) { + assert.ok(doc.includes(m.id), `docs/PROVIDERS.md never mentions ${prov} model ${m.id}`); + } + } +}); + +test('docs/PROVIDERS.md states the per-token vs per-task caveat alongside the table', () => { + const doc = fs.readFileSync(path.join(PKG_ROOT, 'docs', 'PROVIDERS.md'), 'utf8'); + assert.match(doc, PER_TOKEN_QUALIFIER); + assert.match(doc, /per[-\s]task/i); +}); diff --git a/tests/kit/ruvector.test.mjs b/tests/kit/ruvector.test.mjs new file mode 100644 index 0000000..5f5f387 --- /dev/null +++ b/tests/kit/ruvector.test.mjs @@ -0,0 +1,214 @@ +// ruvector — drift management for a global CLI ak deliberately does NOT own. +// +// The whole contract is asymmetric on purpose: ak REPORTS drift and (opt-in) +// upgrades, but never installs. An absent ruvector must therefore produce no +// status row, which is what structurally prevents `ak sync` from installing a +// tool the user never asked for — sync's plan is exactly the rows carrying a +// `fix`, so "no row" is the enforcement mechanism, not just a display choice. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + sandboxHome, assertSandboxed, rmrf, sandboxProject, writeKitConfig, + offlineKitConfig, fakeGlobalRoot, +} from './helpers/home-sandbox.mjs'; + +const HOME = sandboxHome('ak-ruvector'); +const paths = await import('../../src/lib/paths.mjs'); +const ruvector = await import('../../src/lib/ruvector.mjs'); +const status = await import('../../src/commands/status.mjs'); +assertSandboxed(paths, HOME); + +const PKG_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const PROJECT = sandboxProject('ak-ruvector'); + +/** Seed HOME + a fake npm global root. `ruvectorCache` pre-fills the TTL window + * so drift() never reaches npm (the suite must be hermetic and offline). + * `registered` writes the user-scope MCP entry that gates the whole surface — + * ak only manages a ruvector the user actually wired up themselves. */ +function seedHome({ pkgs = {}, ruvectorCache, extra = {}, registered = false } = {}) { + rmrf(paths.claudeDir(), paths.configDir()); + fs.mkdirSync(paths.claudeDir(), { recursive: true }); + fs.writeFileSync(paths.claudeMdPath(), '# machine notes\n'); + const cfg = offlineKitConfig(extra); + if (ruvectorCache) cfg.versionCheck = { ...cfg.versionCheck, ruvector: { last: Date.now(), ...ruvectorCache } }; + writeKitConfig(HOME, cfg); + if (registered) { + fs.writeFileSync(paths.claudeUserMcpPath(), + JSON.stringify({ mcpServers: { ruvector: { command: 'npx', args: ['-y', 'ruvector', 'mcp', 'start'] } } })); + } else { + fs.rmSync(paths.claudeUserMcpPath(), { force: true }); + } + paths._setGlobalRootForTest(fakeGlobalRoot(HOME, { ruflo: '9.9.9', 'agentic-qe': '9.9.9', ...pkgs })); +} + +const collect = () => status.collect({ pkgRoot: PKG_ROOT, cwd: PROJECT }); +const ruvectorRows = (rows) => rows.filter((r) => r.subsystem === 'ruvector'); + +// ── presence detection ────────────────────────────────────────────────────── + +test('present() is false when no global ruvector is installed', () => { + seedHome(); + assert.equal(ruvector.present(), false); +}); + +test('present() is true once ruvector appears in the npm global root', () => { + seedHome({ pkgs: { ruvector: '1.2.0' } }); + assert.equal(ruvector.present(), true); +}); + +test('the managed package name is the bare `ruvector` global', () => { + assert.equal(ruvector.RUVECTOR_PKG, 'ruvector'); +}); + +// ── managed(): the opt-in gate, as one predicate ─────────────────────────── + +test('managed() requires BOTH registration and the kit.json opt-out being unset', () => { + // Registration is the user's signal that they wired ruvector up themselves; + // kit.json is their escape hatch. Either one alone must not enable management. + seedHome({ pkgs: { ruvector: '1.2.0' } }); + assert.equal(ruvector.managed(), false, 'installed but unregistered is not managed'); + + seedHome({ pkgs: { ruvector: '1.2.0' }, registered: true }); + assert.equal(ruvector.managed(), true, 'registered and not opted out'); + + seedHome({ pkgs: { ruvector: '1.2.0' }, registered: true, extra: { ruvector: false } }); + assert.equal(ruvector.managed(), false, 'the kit.json escape hatch wins over registration'); +}); + +test('managed() is false on a machine with no ruvector anywhere', () => { + seedHome(); + assert.equal(ruvector.managed(), false); +}); + +// ── classifyDrift (pure) ──────────────────────────────────────────────────── + +test('classifyDrift reports an absent install as neither present nor outdated', () => { + const d = ruvector.classifyDrift({ installed: null, latest: '2.0.0' }); + assert.equal(d.present, false); + assert.equal(d.outdated, false, 'ak never claims an uninstalled tool is out of date'); +}); + +test('classifyDrift flags a genuinely older install as outdated', () => { + const d = ruvector.classifyDrift({ installed: '1.2.0', latest: '1.3.0' }); + assert.equal(d.present, true); + assert.equal(d.outdated, true); + assert.equal(d.installed, '1.2.0'); + assert.equal(d.latest, '1.3.0'); +}); + +test('classifyDrift treats an unknown latest as unknown, never as up-to-date drift', () => { + // npm unreachable / offline. Reporting `outdated: true` here would plan an + // upgrade on no evidence; reporting a confident "latest" would be a lie. + const d = ruvector.classifyDrift({ installed: '1.2.0', latest: null }); + assert.equal(d.outdated, false); + assert.equal(d.latest, null); +}); + +test('classifyDrift does not flag an install at or ahead of latest', () => { + assert.equal(ruvector.classifyDrift({ installed: '1.3.0', latest: '1.3.0' }).outdated, false); + assert.equal(ruvector.classifyDrift({ installed: '1.4.0', latest: '1.3.0' }).outdated, false, + 'a locally-ahead build is not drift'); +}); + +test('classifyDrift orders versions numerically, not lexically', () => { + // A lexical compare ranks 1.9.0 above 1.10.0 and would silently miss the bump. + assert.equal(ruvector.classifyDrift({ installed: '1.9.0', latest: '1.10.0' }).outdated, true); +}); + +// ── drift() — TTL-cached, and free when ruvector is absent ───────────────── + +test('drift() short-circuits with no network probe when ruvector is absent', async () => { + // PATH in the sandbox contains nothing invokable, so any `npm view` would + // ENOENT — this asserts the absent path never even tries. + seedHome(); + const d = await ruvector.drift(); + assert.equal(d.present, false); + assert.equal(d.latest, null); +}); + +test('drift() serves a fresh TTL cache instead of hitting npm', async () => { + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' } }); + const d = await ruvector.drift(); + assert.equal(d.present, true); + assert.equal(d.installed, '1.2.0'); + assert.equal(d.latest, '1.5.0', 'cached latest used'); + assert.equal(d.outdated, true); +}); + +// ── `ak status` rows ──────────────────────────────────────────────────────── + +test('ak status says NOTHING about an unregistered ruvector, installed or not', async () => { + // The opt-in guarantee. ak manages ruvector only for a user who wired it up as + // an MCP server themselves; registration is that signal. Since sync plans + // exactly the rows carrying a `fix`, "no row" is what makes it structurally + // impossible for ak to install or touch a ruvector nobody asked it to manage. + seedHome(); + assert.deepEqual(ruvectorRows(await collect()), [], 'absent and unregistered'); + + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' } }); + assert.deepEqual(ruvectorRows(await collect()), [], + 'installed but never registered — still none of ak\'s business'); +}); + +test('ak status warns on drift once ruvector is registered, and plans an upgrade', async () => { + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' }, registered: true }); + const [row] = ruvectorRows(await collect()); + assert.ok(row, 'a drifted, registered ruvector must surface'); + assert.equal(row.level, 'warn'); + assert.match(row.message, /1\.2\.0/); + assert.match(row.message, /1\.5\.0/); + assert.ok(row.fix, 'a warn row must tell sync what to do'); + assert.match(row.fix, /upgrade/i, 'and the action is an UPGRADE, never an install'); +}); + +test('ak status reports a current registered ruvector as ok with no planned action', async () => { + seedHome({ pkgs: { ruvector: '1.5.0' }, ruvectorCache: { latest: '1.5.0' }, registered: true }); + const [row] = ruvectorRows(await collect()); + assert.equal(row.level, 'ok'); + assert.equal(row.fix, null, 'nothing to do — sync must not act'); +}); + +test('a registered ruvector with no global CLI is info, never an install plan', async () => { + // The MCP server runs via `npx -y ruvector mcp start`, so registration does not + // imply a global install — and ak must not invent one. + seedHome({ registered: true }); + const [row] = ruvectorRows(await collect()); + assert.ok(row, 'registration is worth reporting'); + assert.equal(row.level, 'info'); + assert.equal(row.fix, null, 'ak never installs the global CLI'); +}); + +test('kit.json ruvector:false downgrades a registered ruvector to an unactionable info row', async () => { + // A user who does not want ak touching their hand-managed CLI keeps the + // visibility but loses the plan entry. + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' }, extra: { ruvector: false }, registered: true }); + const [row] = ruvectorRows(await collect()); + assert.equal(row.level, 'info'); + assert.equal(row.fix, null, 'a disabled subsystem must never be planned by sync'); +}); + +test('a drifted ruvector never plans an INSTALL, only an upgrade of what exists', async () => { + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' }, registered: true }); + const [row] = ruvectorRows(await collect()); + assert.ok(!/\binstall\b/i.test(row.fix), `fix must not offer to install: ${row.fix}`); +}); + +test('the row describes the CLI, never claiming the MCP server runs that version', async () => { + // Honesty pin. The registered command is typically `npx -y ruvector mcp start`, + // so upgrading the global package does not necessarily change what the server + // executes. The row must scope its claim to the CLI it can actually see — + // asserting the server is on that version would be exactly the kind of + // confident-but-wrong signal these two issues were filed about. + seedHome({ pkgs: { ruvector: '1.2.0' }, ruvectorCache: { latest: '1.5.0' }, registered: true }); + const [row] = ruvectorRows(await collect()); + assert.match(row.message, /ruvector CLI/, 'the claim is scoped to the CLI'); + assert.ok(!/server (is )?(on|at|running) 1\.2\.0/i.test(row.message), + `must not claim the MCP server runs the CLI version: ${row.message}`); + assert.ok(!/\bMCP server (is )?(outdated|stale)\b/i.test(row.message), + `must not extend the drift claim to the server: ${row.message}`); +}); + +test.after(() => rmrf(HOME, PROJECT)); diff --git a/tests/kit/status-viability.test.mjs b/tests/kit/status-viability.test.mjs new file mode 100644 index 0000000..5fb2508 --- /dev/null +++ b/tests/kit/status-viability.test.mjs @@ -0,0 +1,310 @@ +// The two "green row that means the wrong thing" defects, at the surface users +// actually read: `ak status`. +// +// #54 — the providers row compared chain ORDER only, so an all-dead chain in +// the right order reported `ok`. Viability is now its own row: `warn`. +// #55 — seeded routing pins diverging from current defaults were invisible. +// Now `info`, deliberately NOT `warn`: measured end-to-end the diverged +// pin is cheaper and faster on routine work, so the user is owed a +// neutral decision, not a lint to clear. +// +// Both defects are the same class — a row that means "matches what's on disk" +// being read as "matches what ak recommends". +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + sandboxHome, assertSandboxed, captureLog, rmrf, sandboxProject, + writeKitConfig, offlineKitConfig, fakeGlobalRoot, +} from './helpers/home-sandbox.mjs'; + +const HOME = sandboxHome('ak-viability'); +const paths = await import('../../src/lib/paths.mjs'); +const status = await import('../../src/commands/status.mjs'); +const sync = await import('../../src/commands/sync.mjs'); +const { AQE_PROVIDER_CREDENTIALS, aqeProviderCredential, credentialGaps } = await import('../../src/lib/providers.mjs'); +const { DEFAULT_ROUTES, ACTIVITIES, seedDualRouting, divergedRoutes } = await import('../../src/lib/routing.mjs'); +assertSandboxed(paths, HOME); + +const PKG_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const PROJECT = sandboxProject('ak-viability'); +const SRC = path.resolve(PKG_ROOT, 'src'); + +// Every env var any credential descriptor consults. Scrubbed around each +// credential assertion so a key exported on the developer's machine can never +// make the "keyless rung" fixture pass or fail by accident. +const ALL_CREDENTIAL_ENV = [...new Set( + Object.values(AQE_PROVIDER_CREDENTIALS).flatMap((d) => [...(d.keyEnv ?? []), ...(d.also ?? [])]), +)]; + +// async + awaited fn(): without the await, `finally` fires the instant fn() +// returns its (still-pending) promise — before any of fn()'s own awaited work +// (e.g. `await collect()`, which itself awaits driftReport() etc.) has run — +// so the restore races ahead and clobbers env vars fn() hasn't read yet. This +// stayed invisible on any machine that happens to export a REAL credential env +// var (e.g. a personal OPENROUTER_API_KEY): the premature restore lands on +// that real value instead of `undefined`, and a real key is just as truthy as +// the test's injected one, so the assertion passes for the wrong reason. +async function withoutEnv(keys, fn) { + const saved = {}; + for (const k of keys) { saved[k] = process.env[k]; delete process.env[k]; } + try { return await fn(); } finally { + for (const k of keys) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + } +} + +function seedHome(extra = {}) { + rmrf(paths.claudeDir(), paths.configDir()); + fs.mkdirSync(paths.claudeDir(), { recursive: true }); + fs.writeFileSync(paths.claudeMdPath(), '# machine notes\n'); + writeKitConfig(HOME, offlineKitConfig(extra)); + paths._setGlobalRootForTest(fakeGlobalRoot(HOME, { ruflo: '9.9.9', 'agentic-qe': '9.9.9' })); +} + +const collect = () => status.collect({ pkgRoot: PKG_ROOT, cwd: PROJECT }); +const rowsFor = (rows, subsystem) => rows.filter((r) => r.subsystem === subsystem); + +/** A non-default provider config (isDefault() must be false or the providers + * branch short-circuits to its advisory row) carrying the given chain. */ +const cfgWithChain = (aqeFallback) => ({ + providers: { hosts: { claude: true, codex: true }, aqeFallback }, +}); + +/** A dual-routing policy pinned to the PRIOR Opus generation — the exact #55 + * reproduction, a machine seeded before the alpha.22 catalog bump. Built from + * seedDualRouting so the escalation ladders are present exactly as a real seed + * writes them: two of the six diverge ONLY on their escalation rung, and a + * hand-built fixture that dropped `escalate` would silently under-report. */ +function divergedPolicy() { + const rewind = (m) => (m === 'claude-opus-5' ? 'claude-opus-4-8' : m); + const seed = seedDualRouting({ hosts: ['claude', 'codex'] }); + const dualRouting = {}; + for (const [act, r] of Object.entries(seed)) { + dualRouting[act] = { + ...r, + model: rewind(r.model), + ...(r.escalate ? { escalate: r.escalate.map((e) => ({ ...e, model: rewind(e.model) })) } : {}), + }; + } + return dualRouting; +} + +/** The six the issue reported, named rather than recomputed — if a catalog bump + * changes this set, the test should fail loudly rather than silently re-derive. */ +const REPRODUCTION_ACTIVITIES = [ + 'architecture', 'debugging', 'design', 'implementation', 'security-analysis', 'testing', +]; + +// ── #54: chain viability is its own row, and it is a WARN ────────────────── + +test('the providers row is WARN when a chain rung has no credential', async () => { + await withoutEnv(ALL_CREDENTIAL_ENV, async () => { + seedHome(cfgWithChain([ + { provider: 'claude-code', models: ['claude-opus-5'] }, + { provider: 'openai', models: ['gpt-5.6'] }, + ])); + const viability = rowsFor(await collect(), 'providers').find((r) => /rungs have credentials/.test(r.message)); + assert.ok(viability, 'chain viability must be reported, not folded into the order check'); + assert.equal(viability.level, 'warn', 'one live rung out of two is not `ok`'); + assert.match(viability.message, /1\/2 rungs have credentials/); + assert.match(viability.message, /openai/, 'the dead rung is named'); + assert.match(viability.message, /OPENAI_API_KEY/, 'and so is the credential that would fix it'); + }); +}); + +test('the credential row is a WARN, never a FAIL — the primary rung still works', async () => { + await withoutEnv(ALL_CREDENTIAL_ENV, async () => { + seedHome(cfgWithChain([ + { provider: 'claude-code', models: ['claude-opus-5'] }, + { provider: 'openai', models: ['gpt-5.6'] }, + ])); + const viability = rowsFor(await collect(), 'providers').find((r) => /rungs have credentials/.test(r.message)); + assert.notEqual(viability.level, 'fail'); + }); +}); + +test('the credential row plans no fix — only the user can supply a key', async () => { + await withoutEnv(ALL_CREDENTIAL_ENV, async () => { + seedHome(cfgWithChain([{ provider: 'openai', models: ['gpt-5.6'] }])); + const viability = rowsFor(await collect(), 'providers').find((r) => /rungs have credentials/.test(r.message)); + assert.equal(viability.fix, null, '`ak sync` cannot invent a credential — it must not plan to'); + }); +}); + +test('the providers row is OK only when every chain rung is viable', async () => { + await withoutEnv(ALL_CREDENTIAL_ENV, async () => { + process.env.OPENROUTER_API_KEY = 'sk-test'; + seedHome(cfgWithChain([ + { provider: 'openrouter', models: ['z-ai/glm-5.2'] }, + { provider: 'ollama', models: ['llama3'] }, + ])); + const viability = rowsFor(await collect(), 'providers').find((r) => /rungs have credentials/.test(r.message)); + assert.equal(viability.level, 'ok'); + assert.match(viability.message, /2\/2 rungs have credentials/); + }); +}); + +// ── the REACHABLE host-credential gap: codex ─────────────────────────────── +// +// provider-credentials.test.mjs covers the host-credential branch by injecting +// a fake hostAuth, but for `claude-code` that state cannot occur in production: +// hostAuthState infers a keychain/subscription login whenever the CLI is +// present, unconditionally, so claude-code never reports a gap. `codex` has no +// such inference — its credential is a real OPENAI_API_KEY or a real +// ~/.codex/auth.json — so it is the host rung that can genuinely go dead. +// +// This must live here rather than beside the other credential tests because it +// reads the filesystem: only a sandboxed HOME makes "no codex login" true +// regardless of whose machine runs the suite. + +test('a codex rung with no key and no login is a REAL, reachable credential gap', () => { + const c = aqeProviderCredential('codex', { env: {} }); + assert.equal(c.known, true); + assert.equal(c.present, false, + 'with HOME sandboxed there is no ~/.codex/auth.json — this is the production-reachable gap'); + assert.ok(c.missing.length > 0, 'and it names what would fix it'); +}); + +test('a codex rung is credentialed by an OPENAI_API_KEY alone', () => { + const c = aqeProviderCredential('codex', { env: { OPENAI_API_KEY: 'sk-test' } }); + assert.equal(c.present, true, 'an api key satisfies codex without any login file'); +}); + +test('the sandbox really has no codex login — this suites premise, asserted', () => { + // Guards the two tests above from silently becoming vacuous (or from passing + // for the wrong reason) if the HOME redirect ever stops taking effect. + assert.ok(!fs.existsSync(path.join(HOME, '.codex', 'auth.json')), + 'the sandboxed HOME must not contain a real codex login'); +}); + +test('credentialGaps reports a dead codex rung through the real env path', () => { + // No injection anywhere: this is the genuine end-to-end shape of a chain whose + // host rung has no credential. + const gaps = credentialGaps([{ provider: 'codex' }, { provider: 'openai' }], { env: {} }); + assert.deepEqual(gaps.map((g) => g.provider), ['codex', 'openai']); +}); + +// ── #55: divergence is INFO, and never framed as lag ─────────────────────── + +// The words a fix must never use. Each asserts the newer default is strictly +// better, which the measurements contradict on routine work — and a user who +// believes it pays 2-3x the agentic turns for nothing. +const FORBIDDEN_FRAMING = /\b(stale|outdated|superseded)\b/i; + +test('diverged seeded routes are reported as INFO, not WARN', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const row = rowsFor(await collect(), 'routing').find((r) => /diverge/i.test(r.message)); + assert.ok(row, 'divergence must surface somewhere — invisibility is the defect'); + assert.equal(row.level, 'info', + '`warn` would push users to spend 2-3x the agentic turns clearing a lint that is sometimes wrong'); +}); + +test('the divergence row never calls a diverged route stale, outdated, or superseded', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const row = rowsFor(await collect(), 'routing').find((r) => /diverge/i.test(r.message)); + assert.ok(!FORBIDDEN_FRAMING.test(row.message), `row must stay neutral, got: ${row.message}`); +}); + +test('the divergence row shows both models so the trade is visible', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const row = rowsFor(await collect(), 'routing').find((r) => /diverge/i.test(r.message)); + assert.match(row.message, /claude-opus-4-8/, 'the pin the machine is on'); + assert.match(row.message, /claude-opus-5/, 'and the default it diverges from'); +}); + +test('the divergence row counts all SIX activities of the reported reproduction', async () => { + // Four diverge on their primary model; implementation/testing diverge only on + // their escalation rung. A row that counted four would under-report the very + // state the issue documented. + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const row = rowsFor(await collect(), 'routing').find((r) => /diverge/i.test(r.message)); + assert.match(row.message, new RegExp(`\\b${REPRODUCTION_ACTIVITIES.length}\\b`)); +}); + +test('divergedRoutes and the status row agree on exactly which six diverge', async () => { + // Pins the set, not just the count — a catalog bump that changes membership + // should fail loudly here rather than silently re-derive a new "correct" answer. + assert.deepEqual(divergedRoutes(divergedPolicy()).map((d) => d.activity).sort(), + REPRODUCTION_ACTIVITIES); +}); + +test('a policy seeded from CURRENT defaults produces no divergence row at all', async () => { + const dualRouting = {}; + for (const act of ACTIVITIES) { + dualRouting[act] = { host: DEFAULT_ROUTES[act].host, model: DEFAULT_ROUTES[act].model, source: 'seeded' }; + } + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting } }); + assert.equal(rowsFor(await collect(), 'routing').filter((r) => /diverge/i.test(r.message)).length, 0); +}); + +test('user-pinned routes are never reported as divergence, however old the model', async () => { + const dualRouting = { architecture: { host: 'claude', model: 'claude-opus-4-8', source: 'user' } }; + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting } }); + assert.equal(rowsFor(await collect(), 'routing').filter((r) => /diverge/i.test(r.message)).length, 0); +}); + +// ── the load-bearing one: `ak sync` must NOT auto-refresh ───────────────── + +test('the divergence row carries no fix — the mechanism that keeps it out of syncs plan', async () => { + // sync's plan is *defined* as the rows carrying a `fix`. A null fix is + // therefore not cosmetic: it is what structurally prevents an auto-refresh. + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const row = rowsFor(await collect(), 'routing').find((r) => /diverge/i.test(r.message)); + assert.equal(row.fix, null); +}); + +test('ak sync never plans a routing refresh for a diverged policy', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const cwd = process.cwd(); + process.chdir(PROJECT); + let out; + try { + ({ out } = await captureLog(() => sync.run({ + flags: { 'dry-run': true, 'no-upgrade': false, json: false }, pkgRoot: PKG_ROOT, + }))); + } finally { process.chdir(cwd); } + const planned = out.split('\n').filter((l) => l.trim().startsWith('•')); + assert.ok(!planned.some((l) => /refresh|diverge/i.test(l)), + `sync must not plan a refresh, got: ${planned.join(' | ')}`); +}); + +test('a dry-run sync leaves the diverged policy byte-identical on disk', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + const before = fs.readFileSync(paths.kitConfigPath(), 'utf8'); + const cwd = process.cwd(); + process.chdir(PROJECT); + try { + await captureLog(() => sync.run({ + flags: { 'dry-run': true, 'no-upgrade': false, json: false }, pkgRoot: PKG_ROOT, + })); + } finally { process.chdir(cwd); } + assert.equal(fs.readFileSync(paths.kitConfigPath(), 'utf8'), before); +}); + +test('sync.mjs does not reference the seeded-route refresh path at all', async () => { + // The behavioral tests above prove sync does not refresh on THESE fixtures. + // This proves it cannot on any fixture: the refresh function is not reachable + // from sync's module. `ak sync` is documented as idempotent reapplication of + // persisted choice — silently changing which model the user's work runs on + // would break that contract (and, per the measurements, cost turns for nothing). + const src = fs.readFileSync(path.join(SRC, 'commands', 'sync.mjs'), 'utf8'); + assert.ok(!/refreshSeededRoutes/.test(src), 'sync must never call refreshSeededRoutes'); + assert.ok(!/divergedRoutes/.test(src), 'sync must not even consult divergence'); +}); + +// ── neutral framing is a whole-surface property, not one row ─────────────── + +test('no status row anywhere frames a diverged route as stale/outdated/superseded', async () => { + seedHome({ providers: { hosts: { claude: true, codex: true }, dualRouting: divergedPolicy() } }); + for (const r of await collect()) { + if (!/diverge|seeded route/i.test(r.message)) continue; + assert.ok(!FORBIDDEN_FRAMING.test(r.message), `neutral framing violated: ${r.message}`); + } +}); + +test.after(() => rmrf(HOME, PROJECT));