feat: publish semantic USB transport profiles

README.md

@@ -141,6 +141,12 @@ client-side in sql.js (WASM). Zero server, fully cacheable.
 
 ### From curl / scripts
 
+Consumers of `usb-profiles.json` must fetch `_meta.json` first, require the
+advertised `usb_profiles_schema_version`, then fetch the artifact and verify
+its SHA-256 equals `usb_profiles_sha256`. A failed or mismatched second fetch
+must be treated as stale/corrupt data; never mix metadata from one build with
+an artifact from another.
+
 Once branches exist, raw JSON is available via:
 
 ```

builders/extract_boards.py

@@ -470,12 +470,20 @@ def _extract_platformio(root: pathlib.Path) -> list[dict]:
         debug = b.get("debug") or {}
 
         vidpids: list[list[str]] = []
+        identity_purposes: dict[str, list[str]] = {}
         for entry in (build.get("hwids") or []):
             if isinstance(entry, (list, tuple)) and len(entry) >= 2:
                 vid = _norm_hex(entry[0], 4)
                 pid = _norm_hex(entry[1], 4)
                 if vid and pid:
                     vidpids.append([vid, pid])
+                    identity_purposes[f"{vid}:{pid}"] = ["runtime"]
+        bvid = _norm_hex(build.get("vid", ""), 4)
+        bpid = _norm_hex(build.get("pid", ""), 4)
+        if bvid and bpid:
+            if [bvid, bpid] not in vidpids:
+                vidpids.append([bvid, bpid])
+            identity_purposes.setdefault(f"{bvid}:{bpid}", []).append("compile")
 
         mcu_str = _str_or_none(build.get("mcu"))
         arch, bits = _derive_arch_bits(mcu_str)
@@ -507,6 +515,7 @@ def _extract_platformio(root: pathlib.Path) -> list[dict]:
             "aliases":          ",".join(aliases) if aliases else None,
             "keywords":         " ".join(sorted(kw_sink)) if kw_sink else None,
             "vidpids":          vidpids,
+            "identity_purposes": identity_purposes,
             "upstream_repo":    repo,
             "upstream_blob":    blob,
             # Source path under the layer's data dir — site.py uses this to
@@ -587,6 +596,7 @@ def _extract_arduino(root: pathlib.Path) -> list[dict]:
             "aliases":          ",".join(aliases) if aliases else None,
             "keywords":         " ".join(sorted(kw_sink)) if kw_sink else None,
             "vidpids":          _arduino_vidpids(b),
+            "identity_purposes": {f"{v}:{p}": ["compile", "runtime"] for v, p in _arduino_vidpids(b)},
             "upstream_repo":    upstream,
             "upstream_blob":    upstream,
             "src_relpath":      f"{core}/boards/{board_id}.json",

builders/extract_other.py

@@ -122,13 +122,15 @@ def extract(root: pathlib.Path) -> dict[str, Any]:
     vid_overrides: dict[str, str] = {}
     vidpid_overrides: dict[str, str] = {}
     vidpid_board_links: list[dict] = []
+    usb_profiles: list[dict] = []
 
     if not root.is_dir():
         print(f"other: input dir {root} missing; emitting empty record set",
               file=sys.stderr)
         return {"layer": "other", "vendors": vendors, "products": products,
                 "vid_overrides": vid_overrides, "vidpid_overrides": vidpid_overrides,
-                "vidpid_board_links": vidpid_board_links}
+                "vidpid_board_links": vidpid_board_links,
+                "usb_profiles": usb_profiles}
 
     ovr_path = root / "overrides.json"
     if ovr_path.is_file():
@@ -157,6 +159,9 @@ def extract(root: pathlib.Path) -> dict[str, Any]:
                 else:
                     print(f"other[skip vidpid_board_link rule #{i}]: malformed",
                           file=sys.stderr)
+            for rec in ovr.get("usb_profiles") or []:
+                if isinstance(rec, dict):
+                    usb_profiles.append(rec)
             print(f"other: {len(vid_overrides)} vid overrides, "
                   f"{len(vidpid_overrides)} vidpid overrides, "
                   f"{len(vidpid_board_links)} board-link rules",
@@ -195,7 +200,8 @@ def extract(root: pathlib.Path) -> dict[str, Any]:
           file=sys.stderr)
     return {"layer": "other", "vendors": vendors, "products": products,
             "vid_overrides": vid_overrides, "vidpid_overrides": vidpid_overrides,
-            "vidpid_board_links": vidpid_board_links}
+            "vidpid_board_links": vidpid_board_links,
+            "usb_profiles": usb_profiles}
 
 
 def main() -> int:

builders/site.py

@@ -38,6 +38,12 @@
 import shutil
 import subprocess
 import sys
+import subprocess as _subprocess
+
+try:  # direct script execution and package imports are both supported
+    from .usb_profiles import artifact_sha256, write_profiles
+except ImportError:
+    from usb_profiles import artifact_sha256, write_profiles
 
 
 HERE = pathlib.Path(__file__).resolve().parent
@@ -54,6 +60,21 @@ def _run_script(script: pathlib.Path, *args: str) -> None:
 def _now_iso() -> str:
     return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
+def _layer_revision(path: pathlib.Path) -> str:
+    for candidate in (path / "_meta.json", path / "data" / "_meta.json"):
+        if candidate.is_file():
+            try:
+                meta = json.loads(candidate.read_text(encoding="utf-8"))
+                value = meta.get("source_revision") or meta.get("commit") or meta.get("sha")
+                if value:
+                    return str(value)
+            except json.JSONDecodeError:
+                pass
+    try:
+        return _subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True, stderr=_subprocess.DEVNULL).strip()
+    except (OSError, _subprocess.CalledProcessError):
+        return "unknown"
+
 
 def _copy_board_jsons(boards: list[dict], data_root: pathlib.Path,
                        public_dir: pathlib.Path) -> int:
@@ -133,6 +154,7 @@ def _clean_public(public_dir: pathlib.Path) -> None:
         "site.db",
         "usb-ids.json",
         "usb-vids.proto.zstd",
+        "usb-profiles.json",
         "_meta.json",
     ):
         f = public_dir / name
@@ -193,6 +215,18 @@ def orchestrate(
         "--out", str(boards_path),
     )
     boards_data = json.loads(boards_path.read_text(encoding="utf-8"))
+    other_data = json.loads((normalized / "other.json").read_text(encoding="utf-8"))
+    layer_revisions = {}
+    for layer in ("vendors", "arduino", "platformio", "other"):
+        layer_revisions[layer] = _layer_revision(data_root / layer)
+        if layer_revisions[layer] == "unknown":
+            raise RuntimeError(f"cannot determine immutable revision for data layer {layer}")
+    for board in boards_data.get("boards") or []:
+        board.setdefault("source_revision", layer_revisions.get(board.get("layer"), "unknown"))
+    for record in other_data.get("usb_profiles") or []:
+        provenance = record.get("provenance")
+        if isinstance(provenance, dict):
+            provenance.setdefault("source_revision", layer_revisions["other"])
 
     # 6b: stage the per-board JSONs as Vite static assets
     boards_copied = _copy_board_jsons(boards_data.get("boards") or [],
@@ -212,9 +246,20 @@ def orchestrate(
         "--db",  str(public_dir / "boards.db"),
         "--out", str(public_dir / "usb-ids.json"),
     )
+    usb_profiles = write_profiles(boards_data.get("boards") or [], other_data,
+                                  public_dir / "usb-profiles.json")
 
     # 8: _meta.json (also written into public/ for Vite to serve)
     merged = json.loads(merged_path.read_text(encoding="utf-8"))
+    source_revisions = {}
+    for layer in ("vendors", "arduino", "platformio", "other"):
+        for candidate in (data_root / layer / "_meta.json", data_root / layer / "data" / "_meta.json"):
+            if candidate.is_file():
+                try:
+                    source_revisions[layer] = json.loads(candidate.read_text(encoding="utf-8"))
+                except json.JSONDecodeError:
+                    pass
+                break
     stats = merged.get("stats", {})
     meta = {
         "schema_version": 5,
@@ -236,6 +281,10 @@ def orchestrate(
         "database":         "boards.db",
         "usb_ids_download": "usb-ids.json",
         "usb_vids_proto_zstd": "usb-vids.proto.zstd",
+        "usb_profiles": "usb-profiles.json",
+        "usb_profiles_schema_version": usb_profiles["schema_version"],
+        "usb_profiles_sha256": artifact_sha256(usb_profiles),
+        "source_revisions": source_revisions,
         "loader":           "vite",
         "boards_root":      "boards/",
     }