From 09a879c58bce9b71d42ea8f07fe5671869042583 Mon Sep 17 00:00:00 2001 From: Petr Date: Sat, 22 Aug 2026 22:45:28 +0200 Subject: [PATCH] test(agent-prompt): gate the documented prompt budget against the enforced one Rebase of #586 onto main after v0.88.0's budget raise (62_000 -> 70_000, #629) and the keboola-expert.md trim (#634). Main's own doc prose (CONTRIBUTING.md, kbagent-pr-reviewer.md) already carries the correct 70 000 B figure and the enforcing-test cross-reference, so nothing there needs to change -- only the drift-guard test itself was still missing. Add test_documented_budget_matches_enforced_budget, which scans BUDGET_DOC_SITES for a " B budget" mention and asserts the number matches PROMPT_BYTE_BUDGET, derived from the constant rather than a second hardcoded literal. That makes a legitimate future budget bump (constant and docs moving together) a no-op for this test, while a doc site left behind still fails it. The regex also recognizes the deprecated "KB" form so a site that regresses to kilobytes is caught, and tolerates space/comma grouping in the digits ("70 000", "70,000", "70000"). --- tests/test_agent_prompt.py | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_agent_prompt.py b/tests/test_agent_prompt.py index 9cc12984..50bcd9ce 100644 --- a/tests/test_agent_prompt.py +++ b/tests/test_agent_prompt.py @@ -12,6 +12,7 @@ from __future__ import annotations +import re from pathlib import Path import pytest @@ -37,8 +38,33 @@ # exhaustive per-command detail belongs in `AGENT_CONTEXT` (loaded on demand), # and the real answer to sustained growth is splitting keboola-expert into # per-domain specialists, not another bump. Trim before you add. +# +# This is the SINGLE SOURCE OF TRUTH for the budget. Prose elsewhere that +# hardcodes its own figure instead of quoting this one drifts silently: +# CONTRIBUTING.md and kbagent-pr-reviewer.md both said "60 KB" long after +# v0.48.0 moved the ceiling to 62 000 B, and both still said "62 000 B" +# after v0.88.0 moved it again to 70 000 B, until each was caught by hand. +# test_documented_budget_matches_enforced_budget below is the gate that +# keeps them honest going forward. PROMPT_BYTE_BUDGET = 70_000 +# Docs that state the prompt budget in prose. Each must quote it as +# " B" (space-grouped, e.g. "70 000 B") next to the +# word "budget" -- see _BUDGET_MENTION_RE below. +BUDGET_DOC_SITES = [ + "CONTRIBUTING.md", + "plugins/kbagent/agents/kbagent-pr-reviewer.md", +] + +# Matches a budget figure immediately followed by "budget" (optionally +# "prompt budget"), in either the enforced byte form ("70 000 B" / "70000 B") +# or the deprecated kilobyte form ("60 KB"). Grouping punctuation (spaces or +# commas) inside the number is tolerated so "70 000", "70,000" and "70000" +# all match -- only the digits are compared against PROMPT_BYTE_BUDGET. +_BUDGET_MENTION_RE = re.compile( + r"(?P\d[\d ,]*\d|\d)\s*(?PKB|B)\s+(?:prompt\s+)?budget" +) + @pytest.fixture(scope="module") def agent_body() -> str: @@ -81,6 +107,42 @@ def test_agent_prompt_under_token_budget(self, agent_body: str) -> None: f"budget is {PROMPT_BYTE_BUDGET} bytes. Trim or split into specialists." ) + def test_documented_budget_matches_enforced_budget(self) -> None: + """Every doc site's budget figure must match PROMPT_BYTE_BUDGET. + + A doc site that hardcodes its own figure instead of quoting this + module's constant is a silent-drift surface: CONTRIBUTING.md and + kbagent-pr-reviewer.md both sat on a stale figure for a full release + span after the constant moved, until someone caught it by hand. The + expected string is derived from PROMPT_BYTE_BUDGET rather than a + second hardcoded literal, so a legitimate future budget change (the + constant and the docs moving together) never fails this test -- + only a doc site left behind does. + + _BUDGET_MENTION_RE also matches the deprecated "KB" form, so a site + that regresses to kilobytes is caught even though its digits happen + to equal PROMPT_BYTE_BUDGET's byte figure (which would never + legitimately occur, but the point is to catch the wrong *unit* too). + """ + expected_bytes = str(PROMPT_BYTE_BUDGET) + for relative_path in BUDGET_DOC_SITES: + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + mentions = _BUDGET_MENTION_RE.findall(text) + assert mentions, ( + f"{relative_path} does not mention the prompt budget in the " + f"expected ' B budget' form. It must quote the " + f"enforced value (PROMPT_BYTE_BUDGET in {Path(__file__).name}), " + f"not its own figure." + ) + for number, unit in mentions: + digits = re.sub(r"[ ,]", "", number) + assert unit == "B" and digits == expected_bytes, ( + f"{relative_path} states the prompt budget as " + f"'{number} {unit}', but PROMPT_BYTE_BUDGET is " + f"{expected_bytes} B ({Path(__file__).name} is the single " + f"source of truth). Update the doc to match." + ) + # --------------------------------------------------------------------- # 2. Non-negotiable rules are all present (plan §6.1 + §14.8)