diff --git a/README.md b/README.md index 2c352b9a1..6dae17432 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ user opts in with `/plugin enable`; an existing install is never flipped by cata - [`desktop-notification`](plugins/desktop-notification) — Alert you when Claude Code needs input — an audible terminal bell, an OSC 9 terminal notification, and an OS-native toast (macOS/Linux) on permission and idle prompts. - [`playbooks`](plugins/playbooks) — Doctrine and knowledge playbooks as on-demand skills, plus a maintainer-facing update skill. boris — Boris Cherny's Claude Code workflow tips (howborisusesclaudecode.com); skill-authoring — Anthropic's internal skill-authoring playbook; fable-5 — Claude Fable 5's operating doctrine (self-authored, no upstream). The boris and skill-authoring packs vendor a verbatim upstream baseline; /playbooks:update drift-checks and syncs those baselines centrally (maintainers). -- [`claude-config`](plugins/claude-config) — Three audit skills for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), and audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability). +- [`claude-config`](plugins/claude-config) — Four audit skills for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), and audit-model-fit (instruction surfaces swept for model-hobbling constraints newer models no longer need). - [`claude-memory`](plugins/claude-memory) — Keeps a repo's Claude Code memory layer healthy and under your control, against criteria derived from official Claude Code documentation. The audit skill checks the instruction/memory layer (CLAUDE.md, CLAUDE.local.md, .claude/rules/, auto-memory) with a deterministic script-backed spine plus judgment-tier checks. The stateless skill inspects, disables, and (confirm-gated) purges Claude-written auto memory across all settings scopes. - [`claude-ops`](plugins/claude-ops) — Claude Code operations toolkit. Seven skills: observability (read locally captured telemetry — OTEL store, collector, hook-event JSONL, ccusage — with trend reports and store pruning), known-issues (search known Claude product GitHub bugs, check service health, maintain a persistent tracked-issue registry), changelog (ingest Claude Code changelog entries and integrate them into the current repo), plugins (bring a machine's plugin fleet current on demand — marketplace refresh, effective-scope updates including in-repo project/local installs, new-plugin install per policy, scope-divergence detection and explicit convergence), morning-brief (read-only gh-based operator morning view — queue-label counts, merge-ready PRs, parked decisions with their RECOMMENDED lines, and loop-lane telemetry freshness), lanes (start/restart/stop/status loop lanes as named background Claude Code sessions seeded from canonical prompt files, with per-lane model/effort and a repo-pull + marketplace-refresh launch step), and a re-runnable setup action that settles where the known-issues registry lives. Plus a family of seven advisory *-audit telemetry-emitter hooks (API errors, config changes, instruction loads, permission denials, pre-compaction, skill usage, tool failures) that emit the shared hook-telemetry envelope, and a reference sink that maps envelopes into the hook-events.jsonl the observability skill reads. - [`skill-quality`](plugins/skill-quality) — Skill-authoring QA tooling: a static contract checker that runs eighteen deterministic checks over a Claude Code skill (frontmatter, listing-budget cap, trigger-keyword preservation, line caps, broken internal refs, markdownlint, gotchas surface, evals presence, precompute opportunity) and a bundled evals.json schema for validation. Runs against any repo's skills directory via the convention-resolution ladder — no baked layout. diff --git a/plugins/claude-config/.claude-plugin/plugin.json b/plugins/claude-config/.claude-plugin/plugin.json index 9339ce5ce..9129dfdf8 100644 --- a/plugins/claude-config/.claude-plugin/plugin.json +++ b/plugins/claude-config/.claude-plugin/plugin.json @@ -1,12 +1,12 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "claude-config", - "version": "0.7.1", - "description": "Three audit skills for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), and audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability).", + "version": "0.8.0", + "description": "Four audit skills for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), and audit-model-fit (instruction surfaces swept for model-hobbling constraints newer models no longer need).", "author": { "name": "Melodic Software", "email": "info@melodicsoftware.com" }, "license": "MIT", - "keywords": ["settings", "configuration", "hooks", "plugins", "permissions", "automation", "audit", "maintenance", "skill"] + "keywords": ["settings", "configuration", "hooks", "plugins", "permissions", "automation", "audit", "maintenance", "skill", "instructions", "model-fit"] } diff --git a/plugins/claude-config/CHANGELOG.md b/plugins/claude-config/CHANGELOG.md index fac83b9eb..b596fa487 100644 --- a/plugins/claude-config/CHANGELOG.md +++ b/plugins/claude-config/CHANGELOG.md @@ -3,6 +3,26 @@ All notable changes to the `claude-config` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.8.0] + +### Added + +- **`audit-model-fit` skill** (`/claude-config:audit-model-fit`). A fourth audit that sweeps the local + Claude Code instruction surfaces — user + project `CLAUDE.md`, skill `SKILL.md` bodies + context + files, agent definitions, `.claude/rules/**`, prompt-type hooks and output styles — for deterministic + constraints that hobble newer, more capable models, and proposes removals/rewrites. Check catalog: + bare prohibitions with no rationale (rewrite to add the *why*, never blanket-delete), over-prescriptive + step lists (cull to intent + hard constraints), over-constraining example blocks (trim toward the + recommended 3–5, not a blanket ban), and stale model-era workarounds — each measured against "would + removing this cause Claude to make mistakes?". A bundled `instruction-surface-scan.sh` enumerates the + surfaces and flags the two grep-able smells as candidates; the judgment stays in the skill body. + **Report-only and human-gated**: it presents findings plus proposed diffs and never edits any + instruction file itself (no `--fix`). Findings inside `melodic-software/standards`-managed + materializations route upstream per the sync-manifest rather than being edited in place. Composes with + (distinct intents, pointers only) `claude-memory:audit` (instruction-layer *health*, same surfaces), + `skill-quality:check` (structure), `docs-hygiene:compress` (token brevity), and the sibling `audit` + (config-file correctness). The plugin `description` now reads "Four audit skills". + ## [0.7.1] ### Changed diff --git a/plugins/claude-config/README.md b/plugins/claude-config/README.md index 535924121..44e4e27bc 100644 --- a/plugins/claude-config/README.md +++ b/plugins/claude-config/README.md @@ -1,6 +1,6 @@ # claude-config -A Claude Code plugin bundling three audit skills for one cohesive capability: keeping a repo's Claude +A Claude Code plugin bundling four audit skills for one cohesive capability: keeping a repo's Claude Code configuration healthy. Each skill answers a different question about the same surface: | Skill | Question it answers | @@ -8,13 +8,17 @@ Code configuration healthy. Each skill answers a different question about the sa | `/claude-config:audit` | Are the configuration FILES (`settings.json`, `settings.local.json`, `.mcp.json`, hooks, plugins, permissions) correct against upstream truth? | | `/claude-config:audit-automation-gaps` | Is the configured automation SET the right set — are there genuine gaps, judged against the enforcement hierarchy? | | `/claude-config:audit-permission-grants` | Are the permission GRANTS (`allowed-tools`, `permissions.allow`) portable and durable — do they survive auto mode, work across machines, and live where they can take effect? | +| `/claude-config:audit-model-fit` | Do the instruction SURFACES (CLAUDE.md, skill bodies, agents, rules, prompt-type hooks) still fit the current model — or do they carry constraints a newer, more capable model no longer needs? | -The instruction/memory layer (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/rules/`, auto-memory) is audited -by the `health` skill in the separate `claude-memory` plugin (see "Migrating from -`claude-config-audit`" below if you relied on the old `memory-health` skill here). +The instruction/memory layer's *health* (`CLAUDE.md`, `CLAUDE.local.md`, `.claude/rules/`, auto-memory) +is audited by the `health` skill in the separate `claude-memory` plugin (see "Migrating from +`claude-config-audit`" below if you relied on the old `memory-health` skill here). `audit-model-fit` +reads the same instruction surfaces but asks a different question — model *fit*, not health: are their +constraints still earning their place on a newer model? All default to report-only; mutations (`--fix`, `--implement`) require explicit opt-in and per-item -user approval. `audit-permission-grants` is report-only (its correct remediation is operator-manual). +user approval. `audit-permission-grants` and `audit-model-fit` are report-only (the former's remediation +is operator-manual; the latter is human-gated by design — it proposes diffs, never applies them). ## What each skill does @@ -63,6 +67,25 @@ Report-only. /claude-config:audit-permission-grants settings # permissions.allow only ``` +### audit-model-fit + +Sweeps the local instruction surfaces (user + project `CLAUDE.md`, skill `SKILL.md` bodies + context +files, agent definitions, `.claude/rules/**`, prompt-type hooks and output styles) for deterministic +constraints that hobble newer, more capable models: bare prohibitions with no rationale, +over-prescriptive step lists, over-constraining example blocks (kept to 3–5, not banned), and stale +model-era workarounds. Each candidate is measured against "would removing this cause Claude to make +mistakes?" A deterministic scan flags the grep-able smells; the judgment stays in the skill body. +Report-only and human-gated — it proposes diffs, never applies them. Findings inside +`melodic-software/standards`-managed materializations route upstream rather than being edited in place. +Distinct from `skill-quality:check` (structure), `docs-hygiene:compress` (token brevity), and `audit` +(config-file correctness). + +```shell +/claude-config:audit-model-fit # sweep all instruction surfaces +/claude-config:audit-model-fit claude-md # CLAUDE.md only +/claude-config:audit-model-fit skills # skill bodies + context files only +``` + ## Consumer conventions The skills read the consuming repo's own `CLAUDE.md` / `.claude/rules/` for project-specific policy: diff --git a/plugins/claude-config/skills/audit-model-fit/SKILL.md b/plugins/claude-config/skills/audit-model-fit/SKILL.md new file mode 100644 index 000000000..1fd772c1d --- /dev/null +++ b/plugins/claude-config/skills/audit-model-fit/SKILL.md @@ -0,0 +1,110 @@ +--- +name: audit-model-fit +description: "Audit local Claude Code instruction surfaces (user + project CLAUDE.md, skill SKILL.md bodies + context files, agent definitions, .claude/rules, prompt-type hooks and output styles) for deterministic constraints that hobble newer, more capable models — bare prohibitions with no rationale, over-prescriptive step lists, over-constraining example blocks, and stale model-era workarounds — and propose removals/rewrites against the bar 'would removing this cause Claude to make mistakes?'. Use when: 'audit model fit', 'unhobble my instructions', 'after a model upgrade', 'are my CLAUDE.md constraints still needed', 'prune stale prompt instructions', 'my skills are too prescriptive'. Report-only: proposes diffs, never auto-applies." +argument-hint: "[scope] — scope: claude-md|skills|agents|rules|hooks|all (default: all)" +user-invocable: true +disable-model-invocation: false +--- + +## Pre-computed context + +Instruction-surface inventory: !`bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-model-fit/scripts/instruction-surface-scan.sh" 2>/dev/null || echo "scan unavailable"` + +## Purpose + +Upgrade-triggered (or on-demand) audit that sweeps the local Claude Code instruction surfaces and +proposes removals/rewrites of deterministic constraints that hobble newer models. Instructions written +to make a prior, weaker model behave — bare prohibitions, mechanical step lists, defensive +over-specification — become carrying cost and can *degrade* a capable model's output. This audit finds +them and proposes changes; a human applies them. + +The governing bar: **"Would removing this instruction cause Claude to make mistakes?"** If no, it is a +removal or loosening candidate. The full check catalog, treatments, sources, and output format live in +[reference/criteria.md](reference/criteria.md) — read it when running this audit. + +## Scope boundary (compose with — route out, do not restate their logic) + +This audit owns instruction **content fit** to the current model. Four adjacent skills own neighboring +questions; route findings that belong to them rather than handling them here: + +- `claude-memory:audit` — instruction-layer **health** against a codified checklist (is `CLAUDE.md` + well-formed, within length, are rules valid). It reads the same surfaces and shares the pruning bar, + but asks "is the instruction layer healthy?"; this audit asks "do its constraints still fit a newer + model?". Route structure/health/length findings there; keep model-fit loosening here. +- `skill-quality:check` — **structural** SKILL.md lint (frontmatter, line caps, broken refs). +- `docs-hygiene:compress` — **token brevity** (shorten prose without dropping meaning). This audit + removes now-unnecessary *constraints*, not just words. +- `claude-config:audit` — **config-file** hygiene (settings.json / .mcp.json / hooks / permissions). + +## Cross-repo routing (standards-managed materializations) + +A finding inside a file **synced from `melodic-software/standards`** (owned upstream per that repo's +`distribution/sync-manifest.yml`) is **never edited in place** — the next sync overwrites it. Route the +change upstream to the owning repo and say so in the report; propose no in-place diff for that file. If +the manifest is not reachable from the consuming repo, still apply the rule: flag any vendored/synced +file for upstream routing. Locally-owned surfaces get the normal human-gated proposed diff. + +## Arguments + +Parse `$ARGUMENTS` for an optional scope filter: + +- `claude-md` — user + project `CLAUDE.md` only +- `skills` — skill `SKILL.md` bodies + `context/`/`reference/` files only +- `agents` — agent definitions only +- `rules` — `.claude/rules/**/*.md` only +- `hooks` — prompt-type hook scripts + output styles only +- `all` — everything (default) + +This skill is **report-only**. There is no `--fix`: every change is human-gated. It proposes diffs; a +human (or a skill the human explicitly invokes) applies them. + +## Phase 1: Enumerate surfaces + +Run the deterministic scan (also injected above at load): + +```shell +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-model-fit/scripts/instruction-surface-scan.sh" +``` + +It reports the surface inventory and flags the two grep-able smells as **candidates** — bare +prohibitions (C1) and example-dense files (C3). `--candidates` lists each `file:line`; `--count` prints +the bare-prohibition count. Candidates are starting points, not verdicts. + +If a scope filter was given, read only the matching surfaces in Phase 2. + +## Phase 2: Classify against the check catalog + +Read [reference/criteria.md](reference/criteria.md) and read the in-scope surfaces. For each candidate, +and each instruction in the surfaces you read, apply the checks: + +- **C1 bare prohibition** → propose a **prohibition + rationale** rewrite, not deletion (the constraint + may still be needed; only the missing *why* is the defect). +- **C2 over-prescriptive step list** → cull to intent + hard constraints; keep steps that encode a real + ordering/safety/contract. +- **C3 example-dense** → trim toward 3–5 format-steering examples; **not** a blanket ban — a file at or + below the range is not a finding. +- **C4 stale model-era workaround** → verbose over-specification compensating for a prior model's + weakness; propose removal/loosening. + +Measure every candidate against the pruning bar. **No** (removal changes nothing) → removal/loosening +candidate. **Yes** → keep it (for a C1, still propose the rationale rewrite). A clean sweep is a valid +outcome — do not manufacture findings. + +## Phase 3: Report + proposed diffs (human-gated) + +Present findings as the table and proposed-diffs format in +[reference/criteria.md](reference/criteria.md) "Output format": one row per finding with its check, +surface, pruning-bar verdict, proposed change, and routing. For each locally-owned finding, give the +exact before/after rewrite. For a standards-managed finding, the routing reads +"upstream: melodic-software/standards" and **no** in-place diff is proposed. + +Then stop. Do **not** edit any `CLAUDE.md`, `SKILL.md`, agent definition, rule, hook, or output style. +Applying the diffs is a human action (or a separate skill the human explicitly invokes): + +> "Here are the proposed changes. Which would you like applied? I will not change any file without your go-ahead." + +## Consumer conventions + +A consuming repo may declare, in its own `CLAUDE.md` / `.claude/rules/`, prohibitions it deliberately +keeps bare, example sets it intends to hold above the 3–5 range, or surfaces exempt from this sweep. +Read those when present; this skill does not assume them. diff --git a/plugins/claude-config/skills/audit-model-fit/evals/evals.json b/plugins/claude-config/skills/audit-model-fit/evals/evals.json new file mode 100644 index 000000000..84f5c9871 --- /dev/null +++ b/plugins/claude-config/skills/audit-model-fit/evals/evals.json @@ -0,0 +1,113 @@ +{ + "skill_name": "audit-model-fit", + "evals": [ + { + "id": 1, + "name": "sweep-happy-path", + "prompt": "I just upgraded my model. Audit my CLAUDE.md and skills for instructions that are hobbling it and tell me what to prune.", + "expected_output": "Runs the instruction-surface-scan to enumerate CLAUDE.md, skills, agents, rules, and hooks/output-styles, classifies candidates against the check catalog (bare prohibitions, over-prescriptive step lists, example-dense blocks, stale model-era workarounds), measures each against 'would removing this cause Claude to make mistakes?', and presents a findings report plus proposed diffs.", + "files": [], + "expectations": [ + "Output runs the instruction-surface-scan.sh scan rather than eyeballing the files", + "Output classifies findings against the check catalog and applies the 'would removing this cause mistakes?' pruning bar", + "Output presents findings plus proposed diffs rather than applying any change" + ] + }, + { + "id": 2, + "name": "bare-prohibition-rewrite-not-delete", + "prompt": "My CLAUDE.md says 'Never commit directly to main.' with no reason given. Is that a finding, and if so should I just delete it?", + "expected_output": "Flags it as a bare-prohibition (C1) smell but recommends a prohibition-plus-rationale REWRITE, not deletion — the constraint may still be needed; only the missing rationale is the defect. Deletion applies only if the pruning bar says removing it changes nothing.", + "files": [], + "expectations": [ + "Output identifies the bare prohibition as a C1 finding", + "Output recommends rewriting it to add a rationale, NOT deleting the prohibition", + "Output does not treat every 'never'/'do not' as a delete candidate" + ] + }, + { + "id": 3, + "name": "examples-not-blanket-flagged", + "prompt": "My skill has 4 example blocks steering the output format. Flag them as over-constraining and tell me to remove them.", + "expected_output": "Does NOT blanket-flag the examples: current guidance still recommends keeping 3-5 format-steering examples, so 4 is within range and not a finding. Only example volume well beyond that range is flagged as over-constraining (C3), and the treatment is trimming toward 3-5, never a blanket removal.", + "files": [], + "expectations": [ + "Output states that 3-5 format-steering examples are recommended to keep and 4 is within range", + "Output does NOT flag the 4 example blocks as a finding or recommend removing them", + "Output explains only example volume well beyond 3-5 is the C3 over-constraining smell" + ] + }, + { + "id": 4, + "name": "standards-managed-routes-upstream", + "prompt": "One of the over-prescriptive rule files I want pruned is a copy synced from melodic-software/standards. Just edit it in place to loosen it.", + "expected_output": "Explains a standards-managed materialization is owned upstream per the sync-manifest and the next sync overwrites a local edit, so it is never edited in place. Routes the change upstream to melodic-software/standards and proposes no in-place diff for that file.", + "files": [], + "expectations": [ + "Output explains the file is a standards-managed materialization that a sync would overwrite", + "Output routes the change upstream to melodic-software/standards instead of editing in place", + "Output does not propose an in-place diff for the standards-managed file" + ] + }, + { + "id": 5, + "name": "never-auto-applies", + "prompt": "Audit my instruction surfaces and just auto-apply every loosening you find so I don't have to review them.", + "expected_output": "The skill is report-only and human-gated: it runs the audit and presents findings plus proposed diffs, but does not edit any CLAUDE.md, SKILL.md, agent, rule, hook, or output style itself. There is no --fix; applying the diffs is a human action.", + "files": [], + "expectations": [ + "Output does not edit any instruction file itself", + "Output presents proposed diffs for human review rather than auto-applying them", + "Output states the skill is report-only / human-gated with no --fix" + ] + }, + { + "id": 6, + "name": "route-out-to-siblings", + "prompt": "While auditing model fit, also validate my SKILL.md frontmatter and line caps, shorten the verbose prose, and check my settings.json permissions for drift.", + "expected_output": "Audits only instruction content-fit and routes the adjacent concerns to their owning skills: structural frontmatter/line-cap lint to skill-quality:check, prose-length shortening to docs-hygiene:compress, and settings.json/permissions drift to claude-config:audit — without restating their logic.", + "files": [], + "expectations": [ + "Output audits only the content-fit question it owns", + "Output routes structural SKILL.md lint to skill-quality:check and config/permissions drift to claude-config:audit", + "Output routes prose shortening to docs-hygiene:compress and distinguishes removing constraints from shortening words" + ] + }, + { + "id": 7, + "name": "step-list-culled-not-preserved", + "prompt": "My skill spells out a 9-step numbered procedure for how to open a PR — 'run git status, then git add, then write the message, then...'. Should I keep all the steps for a capable model?", + "expected_output": "Flags the mechanical step list as a C2 over-prescriptive finding and recommends culling it to the intent plus the hard constraints (ordering that genuinely matters, safety gates, external contracts), removing the steps that only narrate a procedure a capable model already sequences correctly — not preserving all 9 steps verbatim.", + "files": [], + "expectations": [ + "Output identifies the numbered step list as a C2 over-prescriptive-step-list finding", + "Output recommends culling to intent + hard constraints, not keeping all steps verbatim", + "Output notes steps encoding a real ordering/safety/contract are kept" + ] + }, + { + "id": 8, + "name": "stale-workaround-removed", + "prompt": "My CLAUDE.md has a paragraph re-explaining what JSON is and a defensive 'if you are unsure how to format the output, just wrap it in a code block' note, added back when the old model kept getting it wrong. Keep them?", + "expected_output": "Flags both as C4 stale model-era workarounds — verbose over-specification and defensive scaffolding compensating for a prior model's weakness — and, applying the pruning bar (removing them would not cause the current model to make mistakes), proposes removal/loosening rather than retention.", + "files": [], + "expectations": [ + "Output identifies the re-explanation and defensive scaffolding as C4 stale model-era workarounds", + "Output applies the pruning bar and concludes removing them would not cause mistakes on the current model", + "Output proposes removing/loosening them rather than keeping them" + ] + }, + { + "id": 9, + "name": "clean-sweep-valid-outcome", + "prompt": "Run the model-fit audit on my repo — but I've already trimmed everything and every instruction earns its place.", + "expected_output": "Runs the sweep and, if every instruction passes the pruning bar, reports a clean result as a valid outcome rather than manufacturing findings to justify the audit.", + "files": [], + "expectations": [ + "Output runs the scan and applies the pruning bar", + "Output reports a clean sweep as a valid outcome when nothing fails the bar", + "Output does not manufacture findings to justify the audit" + ] + } + ] +} diff --git a/plugins/claude-config/skills/audit-model-fit/reference/criteria.md b/plugins/claude-config/skills/audit-model-fit/reference/criteria.md new file mode 100644 index 000000000..a4be9a7f7 --- /dev/null +++ b/plugins/claude-config/skills/audit-model-fit/reference/criteria.md @@ -0,0 +1,148 @@ +# Model-Fit Criteria + +Version: 1.0.0 +Last updated: 2026-07-21 + +This file defines the checks the `audit-model-fit` audit runs against local Claude Code instruction +surfaces. It sweeps for deterministic constraints that hobble newer, more capable models — instructions +that made a prior, weaker model behave but now only narrow a capable one — and proposes removals or +rewrites. It **never** applies them: output is a findings report plus proposed diffs, human-gated. + +The governing question — the pruning bar every finding is measured against — is from Anthropic's +guidance: **"Would removing this instruction cause Claude to make mistakes?"** If no, the instruction is +a removal or loosening candidate. + +Sources (verify against these; do not fabricate deep-link anchors): + +- [Prompting Claude Fable 5](https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/prompting-claude-fable-5) + — "re-evaluate which instructions, tools, and guardrails are still needed"; skills built for prior + models are "often too prescriptive… and can degrade output quality". +- [Claude prompting best practices](https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/claude-prompting-best-practices) +- [Claude Code best practices](https://code.claude.com/docs/en/best-practices) — the pruning bar. + +## Surfaces + +The deterministic spine is +`bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-model-fit/scripts/instruction-surface-scan.sh"`. It enumerates, +across the consuming repo (and the user-global `~/.claude/CLAUDE.md`): + +- user + project `CLAUDE.md` (`CLAUDE.md`, `.claude/CLAUDE.md`, `CLAUDE.local.md`, `~/.claude/CLAUDE.md`) +- skill `SKILL.md` bodies + their `context/` and `reference/` files +- agent definitions (`.claude/agents/*.md`) +- `.claude/rules/**/*.md` +- output styles (`.claude/output-styles/*.md`) and prompt-type hook scripts (`.claude/hooks`) + +It flags the two grep-able smells (C1 bare prohibitions, C3 example-dense files) as **candidates** and +is advisory (always exits 0). `--candidates` lists each `file:line`; `--count` prints the +bare-prohibition candidate count. The other checks are model judgment applied while reading the surfaces +— the script does not verdict them. + +Prose surfaces (CLAUDE.md, skill bodies + context, agents, rules, output styles) are grep-scanned for +the two candidate smells. **Hook scripts are enumerated (counted) but not grep-scanned** — a prompt-type +hook's instruction text is embedded in code, where a "never/do not" grep is noise; read the hook bodies +by hand in Phase 2 and apply the same checks. + +Findings are reported, never applied. There is no `--fix`. + +--- + +## C1: Bare prohibition — "never/do not X" with no stated why [rewrite, not delete] + +**What**: A prohibition (`never`, `do not`, `must not`, `shall not`) with no rationale attached. The +scan flags the prohibition lines; whether a rationale is present is the model's call. + +**Why it is a smell**: a capable model follows a reasoned constraint and generalizes it; a bare "never +X" it cannot reason about is both easier to misapply and a frequent relic of steering a weaker model. + +**Treatment**: rewrite to **prohibition + rationale**, not deletion. The constraint may still be needed +— only the missing *why* is the defect. Delete only if the pruning bar says removing it changes nothing. + +## C2: Over-prescriptive step list → cull to intent + constraints [rewrite] + +**What**: A mechanical, numbered step-by-step recipe for something a capable model already sequences +correctly on its own. + +**Why it is a smell**: prescriptive step lists built for prior models "can degrade output quality" — they +over-constrain the model's own planning. Keep the *goal* and the *hard constraints*; drop the mechanics. + +**Treatment**: rewrite to state the intent and the non-negotiable constraints, removing steps that only +narrate an obvious procedure. Keep any step that encodes a genuine hard constraint (an ordering that +matters, a safety gate, an external contract). + +## C3: Example-dense block — over-constraining format steering [rewrite; NOT a blanket ban] + +**What**: An instruction surface carrying far more format-steering example blocks than needed. The scan +flags files with **more than 5** fenced/`` blocks as candidates. + +**Why it is a smell — and its limit**: current guidance still recommends keeping **3–5** examples to +steer format; examples are not the enemy. Only volume **well beyond** that range starts pinning the +model to a rigid template. Do **not** blanket-flag example blocks. + +**Treatment**: trim toward the 3–5 that genuinely steer format; keep coverage of the distinct shapes, +drop near-duplicates. A file at or below the range is not a finding. + +## C4: Stale model-era workaround [remove/loosen] + +**What**: An instruction that reads as compensating for a prior, less-capable model's specific weakness — +verbose over-specification of things a capable model now gets right (belaboring output format it would +infer, re-explaining a concept it knows, defensive "if you are unsure, do X" scaffolding for a task it +now handles). + +**Why it is a smell**: it is pure carrying cost — tokens and rigidity spent to fix a model that is no +longer running. + +**Treatment**: apply the pruning bar. If removing it would not cause mistakes on the current model, +propose removal or loosening. + +--- + +## The pruning bar (applies to every finding) + +For each candidate, ask: **would removing this instruction cause Claude to make mistakes?** + +- **No** → removal/loosening candidate (report the proposed diff). +- **Yes** → keep it. If it is a bare prohibition (C1), still propose the rationale rewrite. + +A clean sweep — every instruction earns its place — is a valid outcome. Do not manufacture findings. + +## Compose with — distinct intents, route out (do not restate their logic) + +- `claude-memory:audit` — instruction-layer **health** (well-formed `CLAUDE.md`, length, valid rules) + against a codified checklist. Same surfaces, same pruning bar, different question: health/structure + vs. model-fit. Route health/length/structure findings there; keep constraint-loosening here. +- `skill-quality:check` — **structural** lint of a SKILL.md (frontmatter, line caps, broken refs). This + audit is content/fit, not structure. Structural defects route there. +- `docs-hygiene:compress` — **token brevity** (shorten prose without semantic loss). This audit removes + now-unnecessary *constraints*, not just words. A prohibition that should stay but read tighter is a + compress job; a prohibition that should loosen is this audit. +- `claude-config:audit` — **config-file** hygiene (settings.json / .mcp.json / hooks / permissions). This + audit reads instruction *content*, not config correctness. + +## Cross-repo routing — standards-managed materializations + +A finding inside a file that is a **materialization synced from `melodic-software/standards`** (owned +upstream per that repo's `distribution/sync-manifest.yml`) is **never edited in place** — the local copy +would be overwritten on the next sync and the change lost. Route such a finding upstream to +`melodic-software/standards` (or the owning source repo the manifest names) as the change target, and +say so in the report. Locally-owned surfaces are edited via the normal human-gated proposed diff. When +the manifest is not reachable from the consuming repo, still apply the rule: flag any file that looks +vendored/synced and note it needs upstream routing rather than an in-place edit. + +## Output format + +```text +## Model-Fit Report — {date} + +### Surfaces scanned +(inventory from instruction-surface-scan.sh) + +### Findings +| # | Check | Surface | Finding | Pruning-bar verdict | Proposed change | Routing | +|---|-------|---------|---------|---------------------|-----------------|---------| + +### Proposed diffs +(per finding: the exact before/after rewrite — human applies, never auto-applied) +``` + +For a standards-managed finding, the Routing column reads "upstream: melodic-software/standards" and no +in-place diff is proposed for that file. diff --git a/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.sh b/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.sh new file mode 100755 index 000000000..cc80f6c7e --- /dev/null +++ b/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# Enumerate local Claude Code instruction surfaces and surface the grep-able +# model-fit smells for the audit-model-fit skill. +# +# Scans project + user CLAUDE.md, skill SKILL.md bodies (+ context files), +# agent definitions, .claude/rules/**/*.md, output styles, and hook scripts. +# Flags the two deterministically-detectable smells as CANDIDATES only — +# bare prohibitions (never/do not/must not) and oversized example-block files. +# The judgment (is a rationale present? is a step list over-prescriptive? does +# removing this cause mistakes?) stays in the skill body, never here. +# +# Modes: (default) compact inventory + smell summary +# --candidates list each flagged file:line +# --count print total candidate-line count only +# Exit: always 0 (advisory). +set -u + +usage() { + cat <<'EOF' +instruction-surface-scan.sh — enumerate Claude Code instruction surfaces and +flag the grep-able model-fit smells (bare prohibitions, oversized example blocks). + +Usage: + instruction-surface-scan.sh [--candidates | --count | --help] + +Modes: + (default) compact surface inventory + smell summary + --candidates list each flagged file:line (bounded to 200 lines) + --count print the total bare-prohibition candidate-line count only + +Exit: always 0 (advisory — candidates require model judgment, not a verdict). +EOF +} + +MODE="summary" +case "${1:-}" in +-h | --help) + usage + exit 0 + ;; +--candidates) MODE="candidates" ;; +--count) MODE="count" ;; +"") ;; +*) + usage >&2 + exit 0 + ;; +esac + +repo_root="$(git rev-parse --show-toplevel 2>/dev/null | tr -d '\r')" +[[ -n "$repo_root" ]] && cd "$repo_root" || true + +# Word-boundary bare-prohibition pattern (case-insensitive). A prohibition is +# only a smell when NO rationale accompanies it — that call is the model's; the +# script just locates the prohibitions. +PROHIBITION='(^|[^[:alnum:]])(never|do not|don'\''t|must not|shall not)([^[:alnum:]]|$)' +EXAMPLE_BLOCK_THRESHOLD=5 + +# Collect the prose surface files (NUL-safe) into an array to grep for smells. +surfaces=() + +# CLAUDE_CONFIG_DIR relocates the whole ~/.claude tree; honor it so the scan +# reads the user-global CLAUDE.md actually loaded into the session. +user_config_dir="${CLAUDE_CONFIG_DIR:-$HOME/.claude}" +claude_md_count=0 +for f in CLAUDE.md .claude/CLAUDE.md CLAUDE.local.md "$user_config_dir/CLAUDE.md"; do + if [[ -f "$f" ]]; then + surfaces+=("$f") + claude_md_count=$((claude_md_count + 1)) + fi +done + +skill_count=0 +context_count=0 +if [[ -d .claude/skills ]]; then + while IFS= read -r -d '' f; do + surfaces+=("$f") + skill_count=$((skill_count + 1)) + done < <(find .claude/skills -name 'SKILL.md' -print0 2>/dev/null) + while IFS= read -r -d '' f; do + surfaces+=("$f") + context_count=$((context_count + 1)) + done < <(find .claude/skills \( -path '*/context/*' -o -path '*/reference/*' \) -name '*.md' -print0 2>/dev/null) +fi + +agent_count=0 +while IFS= read -r -d '' f; do + surfaces+=("$f") + agent_count=$((agent_count + 1)) +done < <(find .claude/agents -maxdepth 1 -name '*.md' -print0 2>/dev/null) + +rule_count=0 +while IFS= read -r -d '' f; do + surfaces+=("$f") + rule_count=$((rule_count + 1)) +done < <(find .claude/rules -name '*.md' -print0 2>/dev/null) + +style_count=0 +while IFS= read -r -d '' f; do + surfaces+=("$f") + style_count=$((style_count + 1)) +done < <(find .claude/output-styles -maxdepth 1 -name '*.md' -print0 2>/dev/null) + +# Hook scripts are enumerated (counted) but NOT grep-scanned: a prompt-type hook's +# instruction text is embedded in code, where a "never/do not" grep is noise. The +# model reads the hook bodies by hand in the skill's Phase 2 instead. +hook_count="$(find .claude/hooks -maxdepth 1 -type f 2>/dev/null | wc -l | tr -d ' ')" + +# --- Candidate detection over the collected surfaces ----------------------- +prohibition_lines=0 +prohibition_files=0 +oversized_example_files=0 +candidate_out="" + +for f in "${surfaces[@]}"; do + # grep -c prints the count (0 on no match) and exits 1 on zero matches; the + # exit code is irrelevant here, so no `|| echo 0` (which would append a second + # line and break the arithmetic below). + hits="$(grep -icE "$PROHIBITION" "$f" 2>/dev/null)" + hits="${hits:-0}" + if ((hits > 0)); then + prohibition_lines=$((prohibition_lines + hits)) + prohibition_files=$((prohibition_files + 1)) + if [[ "$MODE" == "candidates" ]]; then + while IFS= read -r line; do + candidate_out+="$f:$line"$'\n' + done < <(grep -inE "$PROHIBITION" "$f" 2>/dev/null | cut -c1-200) + fi + fi + fences="$(grep -cE '^[[:space:]]*```' "$f" 2>/dev/null)" + tags="$(grep -cE '' "$f" 2>/dev/null)" + blocks=$(((${fences:-0} / 2) + ${tags:-0})) + if ((blocks > EXAMPLE_BLOCK_THRESHOLD)); then + oversized_example_files=$((oversized_example_files + 1)) + if [[ "$MODE" == "candidates" ]]; then + candidate_out+="$f:0: EXAMPLE-DENSE ($blocks blocks > $EXAMPLE_BLOCK_THRESHOLD)"$'\n' + fi + fi +done + +if [[ "$MODE" == "count" ]]; then + printf '%s\n' "$prohibition_lines" + exit 0 +fi + +if [[ "$MODE" == "candidates" ]]; then + if [[ -z "$candidate_out" ]]; then + printf 'No grep-able candidates found.\n' + else + printf '%s' "$candidate_out" | head -n 200 + fi + exit 0 +fi + +cat <$EXAMPLE_BLOCK_THRESHOLD fenced/ blocks): $oversized_example_files files + +Over-prescriptive step lists, stale model-era workarounds, and the +"would removing this cause mistakes?" bar are model judgment — not scanned here. +EOF diff --git a/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.test.sh b/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.test.sh new file mode 100755 index 000000000..fb454ab34 --- /dev/null +++ b/plugins/claude-config/skills/audit-model-fit/scripts/instruction-surface-scan.test.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tests for instruction-surface-scan.sh (self-contained — ships with the plugin). +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCAN="$SCRIPT_DIR/instruction-surface-scan.sh" + +FAILED=0 +CASE_NUM=0 + +pass() { + CASE_NUM=$((CASE_NUM + 1)) + printf 'PASS: %s\n' "$1" +} +fail() { + CASE_NUM=$((CASE_NUM + 1)) + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2 +} +assert_exit() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected exit $2, got $3"; fi +} +assert_contains() { + case "$2" in + *"$3"*) pass "$1" ;; + *) fail "$1" "expected to contain: $3" ;; + esac +} +assert_equals() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected '$2', got '$3'"; fi +} + +rc=0 +bash "$SCAN" --help >/dev/null 2>&1 || rc=$? +assert_exit "--help exits 0" 0 "$rc" + +# Build a throwaway fixture repo with seeded surfaces and smells. +FIX="$(mktemp -d)" +trap 'rm -rf "$FIX"' EXIT +( + cd "$FIX" || exit 1 + git init -q + mkdir -p .claude/skills/demo/context .claude/agents .claude/rules + cat >CLAUDE.md <<'MD' +Never force-push to main. +Do not skip the tests. +This line explains a rationale and is fine. +MD + printf 'name: demo\n' >.claude/skills/demo/SKILL.md + cat >.claude/skills/demo/context/examples.md <<'MD' +``` +a +``` +``` +b +``` +``` +c +``` +``` +d +``` +``` +e +``` +``` +f +``` +MD + # Exactly 5 blocks — at the recommended ceiling, must NOT be flagged (C3 boundary). + cat >.claude/skills/demo/context/at-threshold.md <<'MD' +``` +a +``` +``` +b +``` +``` +c +``` +``` +d +``` +``` +e +``` +MD + printf 'agent body\n' >.claude/agents/x.md + printf 'a rule\n' >.claude/rules/y.md +) + +# Isolate HOME and CLAUDE_CONFIG_DIR so the real user CLAUDE.md never leaks into +# the fixture scan (CLAUDE_CONFIG_DIR points at a fixture path with no CLAUDE.md). +rc=0 +out="$(cd "$FIX" && HOME="$FIX" CLAUDE_CONFIG_DIR="$FIX/.claude" bash "$SCAN" 2>/dev/null)" || rc=$? +assert_exit "default mode exits 0" 0 "$rc" +assert_contains "reports CLAUDE.md surface" "$out" "CLAUDE.md files: 1" +assert_contains "reports agent surface" "$out" "Agent definitions: 1" +assert_contains "reports rule surface" "$out" "Rule files (.claude/rules): 1" +assert_contains "flags bare prohibitions" "$out" "Bare-prohibition lines" +assert_contains "flags example-dense file" "$out" "Example-dense files (>5" +# Only the 6-block file counts — the 5-block file at the ceiling is NOT flagged (C3 boundary). +assert_contains "5-block file not flagged; only the 6-block one" "$out" "blocks): 1 files" + +cnt="$(cd "$FIX" && HOME="$FIX" CLAUDE_CONFIG_DIR="$FIX/.claude" bash "$SCAN" --count 2>/dev/null)" +assert_equals "count matches seeded prohibitions" "2" "$cnt" + +# CLAUDE_CONFIG_DIR relocates the user config tree — the scan must read the +# user-global CLAUDE.md from there, not a hardcoded $HOME/.claude. +CFG="$(mktemp -d)" +printf 'Never bypass the config-dir override.\n' >"$CFG/CLAUDE.md" +rc=0 +out="$(cd "$FIX" && HOME="$FIX" CLAUDE_CONFIG_DIR="$CFG" bash "$SCAN" 2>/dev/null)" || rc=$? +assert_exit "CLAUDE_CONFIG_DIR mode exits 0" 0 "$rc" +assert_contains "honors CLAUDE_CONFIG_DIR for user CLAUDE.md" "$out" "CLAUDE.md files: 2" +rm -rf "$CFG" + +cand="$(cd "$FIX" && HOME="$FIX" CLAUDE_CONFIG_DIR="$FIX/.claude" bash "$SCAN" --candidates 2>/dev/null)" +assert_contains "candidates list the CLAUDE.md prohibition line" "$cand" "CLAUDE.md:1" +assert_contains "candidates flag the example-dense file" "$cand" "examples.md:0: EXAMPLE-DENSE" +case "$cand" in +*at-threshold.md*EXAMPLE-DENSE*) fail "5-block at-threshold.md must not be flagged" "found in candidates" ;; +*) pass "5-block at-threshold.md absent from candidates" ;; +esac + +# Empty repo: still exits 0 with a clean inventory (no crash on no surfaces). +EMPTY="$(mktemp -d)" +(cd "$EMPTY" && git init -q) +rc=0 +out="$(cd "$EMPTY" && HOME="$EMPTY" CLAUDE_CONFIG_DIR="$EMPTY/.claude" bash "$SCAN" 2>/dev/null)" || rc=$? +assert_exit "empty repo exits 0" 0 "$rc" +assert_contains "empty repo zero prohibitions" "$out" "0 across 0 files" +rm -rf "$EMPTY" + +if [[ "$FAILED" -eq 0 ]]; then + printf '\nAll %d checks passed.\n' "$CASE_NUM" + exit 0 +fi +printf '\n%d/%d checks failed.\n' "$FAILED" "$CASE_NUM" >&2 +exit 1