diff --git a/tests/test_agent_prompt.py b/tests/test_agent_prompt.py index 9cc12984..50bcd9ce 100644 --- a/tests/test_agent_prompt.py +++ b/tests/test_agent_prompt.py @@ -12,6 +12,7 @@ from __future__ import annotations +import re from pathlib import Path import pytest @@ -37,8 +38,33 @@ # exhaustive per-command detail belongs in `AGENT_CONTEXT` (loaded on demand), # and the real answer to sustained growth is splitting keboola-expert into # per-domain specialists, not another bump. Trim before you add. +# +# This is the SINGLE SOURCE OF TRUTH for the budget. Prose elsewhere that +# hardcodes its own figure instead of quoting this one drifts silently: +# CONTRIBUTING.md and kbagent-pr-reviewer.md both said "60 KB" long after +# v0.48.0 moved the ceiling to 62 000 B, and both still said "62 000 B" +# after v0.88.0 moved it again to 70 000 B, until each was caught by hand. +# test_documented_budget_matches_enforced_budget below is the gate that +# keeps them honest going forward. PROMPT_BYTE_BUDGET = 70_000 +# Docs that state the prompt budget in prose. Each must quote it as +# " B" (space-grouped, e.g. "70 000 B") next to the +# word "budget" -- see _BUDGET_MENTION_RE below. +BUDGET_DOC_SITES = [ + "CONTRIBUTING.md", + "plugins/kbagent/agents/kbagent-pr-reviewer.md", +] + +# Matches a budget figure immediately followed by "budget" (optionally +# "prompt budget"), in either the enforced byte form ("70 000 B" / "70000 B") +# or the deprecated kilobyte form ("60 KB"). Grouping punctuation (spaces or +# commas) inside the number is tolerated so "70 000", "70,000" and "70000" +# all match -- only the digits are compared against PROMPT_BYTE_BUDGET. +_BUDGET_MENTION_RE = re.compile( + r"(?P\d[\d ,]*\d|\d)\s*(?PKB|B)\s+(?:prompt\s+)?budget" +) + @pytest.fixture(scope="module") def agent_body() -> str: @@ -81,6 +107,42 @@ def test_agent_prompt_under_token_budget(self, agent_body: str) -> None: f"budget is {PROMPT_BYTE_BUDGET} bytes. Trim or split into specialists." ) + def test_documented_budget_matches_enforced_budget(self) -> None: + """Every doc site's budget figure must match PROMPT_BYTE_BUDGET. + + A doc site that hardcodes its own figure instead of quoting this + module's constant is a silent-drift surface: CONTRIBUTING.md and + kbagent-pr-reviewer.md both sat on a stale figure for a full release + span after the constant moved, until someone caught it by hand. The + expected string is derived from PROMPT_BYTE_BUDGET rather than a + second hardcoded literal, so a legitimate future budget change (the + constant and the docs moving together) never fails this test -- + only a doc site left behind does. + + _BUDGET_MENTION_RE also matches the deprecated "KB" form, so a site + that regresses to kilobytes is caught even though its digits happen + to equal PROMPT_BYTE_BUDGET's byte figure (which would never + legitimately occur, but the point is to catch the wrong *unit* too). + """ + expected_bytes = str(PROMPT_BYTE_BUDGET) + for relative_path in BUDGET_DOC_SITES: + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + mentions = _BUDGET_MENTION_RE.findall(text) + assert mentions, ( + f"{relative_path} does not mention the prompt budget in the " + f"expected ' B budget' form. It must quote the " + f"enforced value (PROMPT_BYTE_BUDGET in {Path(__file__).name}), " + f"not its own figure." + ) + for number, unit in mentions: + digits = re.sub(r"[ ,]", "", number) + assert unit == "B" and digits == expected_bytes, ( + f"{relative_path} states the prompt budget as " + f"'{number} {unit}', but PROMPT_BYTE_BUDGET is " + f"{expected_bytes} B ({Path(__file__).name} is the single " + f"source of truth). Update the doc to match." + ) + # --------------------------------------------------------------------- # 2. Non-negotiable rules are all present (plan §6.1 + §14.8)