From 132742e3cdd7f9e464c81154355a25ed50860fb1 Mon Sep 17 00:00:00 2001 From: CHAK Saray Date: Thu, 3 Sep 2026 07:17:05 +0700 Subject: [PATCH] feat: framework_sources -- record which framework version a mapping was made against (#247) --- CONTRIBUTING.md | 1 + schema/ave-record-1.1.0.schema.json | 39 ++++++++++ schema/ave-record.schema.json | 39 ++++++++++ scripts/check_framework_sources.py | 107 ++++++++++++++++++++++++++++ tests/test_framework_sources.py | 102 ++++++++++++++++++++++++++ 5 files changed, 288 insertions(+) create mode 100644 scripts/check_framework_sources.py create mode 100644 tests/test_framework_sources.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b782348..9159248 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,6 +158,7 @@ python scripts/check_confidence_signal.py # soft-warns on #98 high-confidence r python scripts/write_verification_basis.py # derives verification_basis; reports declarations its axes refute python scripts/check_vulnerability_taxonomy.py # soft-warns on records missing security_boundary/missing_control/vulnerability_rationale python scripts/check_vulnerability_taxonomy.py --strict --only AVE-2026-NNNNN # your new record must carry all three taxonomy fields +python scripts/check_framework_sources.py --strict --only AVE-2026-NNNNN # if your record carries owasp_mcp/owasp_asi/mitre_atlas/nist_ai_rmf, each needs a framework_sources entry pytest tests/ -x -q # full suite: schema, AIVSS arithmetic, mitigation enums ``` diff --git a/schema/ave-record-1.1.0.schema.json b/schema/ave-record-1.1.0.schema.json index a076b11..ae0eea3 100644 --- a/schema/ave-record-1.1.0.schema.json +++ b/schema/ave-record-1.1.0.schema.json @@ -599,6 +599,45 @@ "impact": { "type": "string" } }, "description": "The explicit three-line artifact from audit question Q7: what the component can do, what specific condition makes that dangerous, and what results. Optional. This is the sharpest single check against capability/vulnerability conflation, a record that can't fill this in honestly is probably miscategorized." + }, + "framework_sources": { + "type": "object", + "description": "Which version of each referenced framework this record's mappings (owasp_mcp, owasp_asi, mitre_atlas, nist_ai_rmf) were made against. Optional. A mapping to an unratified or moving framework is undecidable without this: a consumer holding owasp_mcp: [\"MCP03\"] cannot tell which reading of the numbering produced it — confirmed as a live, present-tense divergence, not a hypothetical, in crosswalks/ramparts-to-ave.json, where AVE's own MCP03 and Ramparts' MCP03 are unrelated categories that happen to share a number. Mirrors the commit/pin_status pinning already used on crosswalk endpoints (schema/crosswalk-1.0.0.schema.json), one layer down: a crosswalk endpoint pins the tree a record count was read from, this pins the framework reading a record's own tag was read from. Keyed by the mapping field it describes, e.g. framework_sources.owasp_mcp, not four parallel sibling fields, because frameworks version differently (MCP Top 10 has no release, MITRE ATLAS versions discretely, NIST AI RMF is a dated publication) and a single container shape has to accommodate all of them. See OWASP/www-project-mcp-top-10#52 for the case that prompted this.", + "additionalProperties": { + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "Version or dated publication of the referenced framework this mapping was made against, e.g. '4.9.0' for MITRE ATLAS or a dated NIST AI RMF publication. Use commit instead where the framework is tracked in a repository." + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "description": "Full 40-character commit sha pinning the tree this mapping was read against, where the referenced framework has git history to pin. Same convention as crosswalk endpoint commit pinning: full, not abbreviated." + }, + "read_date": { + "type": "string", + "format": "date", + "description": "ISO 8601 date this framework reading was last verified against. Required when pin_status is unpinnable, where it is the nearest thing an unversionable source has to a pin." + }, + "pin_status": { + "type": "string", + "enum": [ + "unpinnable" + ], + "description": "Present only to declare that the referenced framework has no version, tag, or commit to pin against, e.g. the MCP Top 10 prior to a canonical numbering reference existing. Distinguishes a stated exemption from a field nobody has filled in yet. An entry declaring this should also carry read_date and unpinnable_reason." + }, + "unpinnable_reason": { + "type": "string", + "description": "Why this framework reading cannot be pinned, in terms a reader can check, e.g. 'MCP Top 10 is in pilot with no canonical numbering reference published (OWASP/www-project-mcp-top-10#52)'. Required when pin_status is unpinnable." + }, + "content_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$", + "description": "Lowercase sha256, prefixed sha256:, of the bytes read at read_date, for a framework with no git history to pin. Does not let a reader re-derive the mapping, but does let them tell whether the source has moved since it was read. Optional even when pin_status is unpinnable." + } + } + } } } } diff --git a/schema/ave-record.schema.json b/schema/ave-record.schema.json index a076b11..ae0eea3 100644 --- a/schema/ave-record.schema.json +++ b/schema/ave-record.schema.json @@ -599,6 +599,45 @@ "impact": { "type": "string" } }, "description": "The explicit three-line artifact from audit question Q7: what the component can do, what specific condition makes that dangerous, and what results. Optional. This is the sharpest single check against capability/vulnerability conflation, a record that can't fill this in honestly is probably miscategorized." + }, + "framework_sources": { + "type": "object", + "description": "Which version of each referenced framework this record's mappings (owasp_mcp, owasp_asi, mitre_atlas, nist_ai_rmf) were made against. Optional. A mapping to an unratified or moving framework is undecidable without this: a consumer holding owasp_mcp: [\"MCP03\"] cannot tell which reading of the numbering produced it — confirmed as a live, present-tense divergence, not a hypothetical, in crosswalks/ramparts-to-ave.json, where AVE's own MCP03 and Ramparts' MCP03 are unrelated categories that happen to share a number. Mirrors the commit/pin_status pinning already used on crosswalk endpoints (schema/crosswalk-1.0.0.schema.json), one layer down: a crosswalk endpoint pins the tree a record count was read from, this pins the framework reading a record's own tag was read from. Keyed by the mapping field it describes, e.g. framework_sources.owasp_mcp, not four parallel sibling fields, because frameworks version differently (MCP Top 10 has no release, MITRE ATLAS versions discretely, NIST AI RMF is a dated publication) and a single container shape has to accommodate all of them. See OWASP/www-project-mcp-top-10#52 for the case that prompted this.", + "additionalProperties": { + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "Version or dated publication of the referenced framework this mapping was made against, e.g. '4.9.0' for MITRE ATLAS or a dated NIST AI RMF publication. Use commit instead where the framework is tracked in a repository." + }, + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "description": "Full 40-character commit sha pinning the tree this mapping was read against, where the referenced framework has git history to pin. Same convention as crosswalk endpoint commit pinning: full, not abbreviated." + }, + "read_date": { + "type": "string", + "format": "date", + "description": "ISO 8601 date this framework reading was last verified against. Required when pin_status is unpinnable, where it is the nearest thing an unversionable source has to a pin." + }, + "pin_status": { + "type": "string", + "enum": [ + "unpinnable" + ], + "description": "Present only to declare that the referenced framework has no version, tag, or commit to pin against, e.g. the MCP Top 10 prior to a canonical numbering reference existing. Distinguishes a stated exemption from a field nobody has filled in yet. An entry declaring this should also carry read_date and unpinnable_reason." + }, + "unpinnable_reason": { + "type": "string", + "description": "Why this framework reading cannot be pinned, in terms a reader can check, e.g. 'MCP Top 10 is in pilot with no canonical numbering reference published (OWASP/www-project-mcp-top-10#52)'. Required when pin_status is unpinnable." + }, + "content_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$", + "description": "Lowercase sha256, prefixed sha256:, of the bytes read at read_date, for a framework with no git history to pin. Does not let a reader re-derive the mapping, but does let them tell whether the source has moved since it was read. Optional even when pin_status is unpinnable." + } + } + } } } } diff --git a/scripts/check_framework_sources.py b/scripts/check_framework_sources.py new file mode 100644 index 0000000..4f39e83 --- /dev/null +++ b/scripts/check_framework_sources.py @@ -0,0 +1,107 @@ +# What: reports which of a record's framework mapping fields (owasp_mcp, +# owasp_asi, mitre_atlas, nist_ai_rmf) lack a corresponding +# framework_sources entry recording what version of that framework +# the mapping was made against. A soft warning by default; --strict +# makes it a hard failure, intended for gating new record submissions +# specifically, not the existing corpus. +# Why: a mapping to an unratified or moving framework is undecidable +# without this -- OWASP/www-project-mcp-top-10#52 documents two +# independent projects (and, per crosswalks/ramparts-to-ave.json, +# AVE and Ramparts specifically) assigning the same MCP category +# number to unrelated categories, because each read the spec at a +# different point while it was still moving. The 80 records that +# predate this field are not retroactively required to carry it -- +# determining what each was actually mapped against is real, +# per-record judgment and a separate backfill task (issue #245). +# This check therefore defaults to --only-scoped use gating new +# submissions; corpus-wide CI enforcement is deliberately deferred +# until after the backfill, following the volume caution raised on +# check_confidence_signal.py in #242 (one line to nine on 80 +# records; this field is unset on all 80, so an un-scoped corpus-wide +# CI step today would immediately be noisier than that). +import argparse +import json +import sys +from pathlib import Path + +RECORDS_DIR = Path("records") +FRAMEWORK_FIELDS = ("owasp_mcp", "owasp_asi", "mitre_atlas", "nist_ai_rmf") + + +def has_real_source(entry: dict) -> bool: + """True when a framework_sources entry is a real, checkable pin rather + than an empty or partial placeholder. An unpinnable declaration counts + only with its read_date (the nearest thing an unversionable source has + to a pin); anything else needs a version or commit alongside its + read_date, matching the same pin_status vocabulary already used on + crosswalk endpoints (schema/crosswalk-1.0.0.schema.json). + """ + if not entry: + return False + if entry.get("pin_status") == "unpinnable": + return bool(entry.get("read_date")) + return bool((entry.get("version") or entry.get("commit")) and entry.get("read_date")) + + +def missing_sources(record: dict) -> list: + """Framework fields this record carries a real mapping for for which + framework_sources has no corresponding real entry. + """ + sources = record.get("framework_sources") or {} + missing = [] + for field in FRAMEWORK_FIELDS: + if not record.get(field): + continue + if not has_real_source(sources.get(field)): + missing.append(field) + return missing + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Report AVE records whose framework mapping fields " + "(owasp_mcp, owasp_asi, mitre_atlas, nist_ai_rmf) lack a " + "corresponding framework_sources entry." + ) + parser.add_argument( + "--strict", action="store_true", + help="hard-fail on any checked record missing a framework_sources " + "entry for a mapping it carries, intended for gating new " + "record submissions rather than the existing corpus", + ) + parser.add_argument( + "--only", metavar="AVE_ID", action="append", + help="check only the named record(s), e.g. for a new-record PR gate " + "that shouldn't re-flag the other records", + ) + args = parser.parse_args(argv) + + paths = sorted(RECORDS_DIR.glob("AVE-*.json")) + if not paths: + print(f"No records found under {RECORDS_DIR}/", file=sys.stderr) + return 2 + + findings = {} + for path in paths: + record = json.loads(path.read_text(encoding="utf-8")) + rid = record.get("ave_id", path.stem) + if args.only and rid not in args.only: + continue + missing = missing_sources(record) + if missing: + findings[rid] = missing + + checked = len(args.only) if args.only else len(paths) + if findings: + label = "FAIL" if args.strict else "WARNING" + detail = "; ".join(f"{rid} ({', '.join(fields)})" for rid, fields in findings.items()) + print(f"{label}: {len(findings)} of {checked} record(s) carry a framework " + f"mapping with no corresponding framework_sources entry: {detail}") + return 1 if args.strict else 0 + print(f"All {checked} checked record(s) have framework_sources coverage for " + f"every framework mapping they carry.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_framework_sources.py b/tests/test_framework_sources.py new file mode 100644 index 0000000..473e171 --- /dev/null +++ b/tests/test_framework_sources.py @@ -0,0 +1,102 @@ +import json +import pytest +from scripts import check_framework_sources as check + + +def record(**overrides): + base = {"ave_id": "AVE-2026-99999"} + base.update(overrides) + return base + + +def test_record_with_no_framework_fields_has_nothing_missing(): + assert check.missing_sources(record()) == [] + + +def test_mapped_field_with_no_framework_sources_entry_is_missing(): + assert check.missing_sources(record(owasp_mcp=["MCP03"])) == ["owasp_mcp"] + + +def test_mapped_field_with_real_source_entry_is_not_missing(): + r = record( + owasp_mcp=["MCP03"], + framework_sources={"owasp_mcp": {"commit": "a" * 40, "read_date": "2026-08-20"}}, + ) + assert check.missing_sources(r) == [] + + +def test_source_entry_missing_read_date_does_not_count(): + """Mutation check: if the read_date requirement were dropped, this + must go red. A version/commit alone with no date isn't a real pin.""" + r = record( + owasp_mcp=["MCP03"], + framework_sources={"owasp_mcp": {"commit": "a" * 40}}, + ) + assert check.missing_sources(r) == ["owasp_mcp"] + + +def test_unpinnable_with_read_date_counts_as_a_real_source(): + r = record( + owasp_mcp=["MCP03"], + framework_sources={"owasp_mcp": {"pin_status": "unpinnable", "read_date": "2026-08-20"}}, + ) + assert check.missing_sources(r) == [] + + +def test_unpinnable_without_read_date_does_not_count(): + """Mutation check: if the unpinnable branch stopped checking read_date, + this must go red -- unpinnable alone is a bare declaration, not a pin.""" + r = record( + owasp_mcp=["MCP03"], + framework_sources={"owasp_mcp": {"pin_status": "unpinnable"}}, + ) + assert check.missing_sources(r) == ["owasp_mcp"] + + +def test_multiple_mapped_fields_each_checked_independently(): + r = record( + owasp_mcp=["MCP03"], + mitre_atlas=["AML.T0051.001"], + framework_sources={"owasp_mcp": {"commit": "a" * 40, "read_date": "2026-08-20"}}, + ) + assert check.missing_sources(r) == ["mitre_atlas"] + + +def test_empty_list_field_is_not_treated_as_a_carried_mapping(): + """An empty owasp_asi: [] should not demand a framework_sources entry -- + there's no mapping to have provenance for.""" + assert check.missing_sources(record(owasp_asi=[])) == [] + + +def test_default_mode_warns_and_exits_zero(tmp_path, monkeypatch, capsys): + (tmp_path / "AVE-2026-99999.json").write_text( + json.dumps(record(owasp_mcp=["MCP03"])), encoding="utf-8" + ) + monkeypatch.setattr(check, "RECORDS_DIR", tmp_path) + monkeypatch.setattr("sys.argv", ["check_framework_sources.py"]) + assert check.main([]) == 0 + assert "WARNING" in capsys.readouterr().out + + +def test_strict_mode_fails_on_missing_sources(tmp_path, monkeypatch): + (tmp_path / "AVE-2026-99999.json").write_text( + json.dumps(record(owasp_mcp=["MCP03"])), encoding="utf-8" + ) + monkeypatch.setattr(check, "RECORDS_DIR", tmp_path) + assert check.main(["--strict"]) == 1 + + +def test_only_flag_scopes_the_check_to_named_records(tmp_path, monkeypatch): + """The property a new-record PR gate depends on: --only must not + re-flag the records that predate this field.""" + (tmp_path / "AVE-2026-00001.json").write_text( + json.dumps(record(ave_id="AVE-2026-00001", owasp_mcp=["MCP03"])), encoding="utf-8" + ) + complete = record( + ave_id="AVE-2026-99999", + owasp_mcp=["MCP03"], + framework_sources={"owasp_mcp": {"commit": "a" * 40, "read_date": "2026-08-20"}}, + ) + (tmp_path / "AVE-2026-99999.json").write_text(json.dumps(complete), encoding="utf-8") + monkeypatch.setattr(check, "RECORDS_DIR", tmp_path) + assert check.main(["--strict", "--only", "AVE-2026-99999"]) == 0