feat(enrichment): unify binary extension inventory for asset-weight and provenance

review-enrichment/src/analyzers/asset-weight.ts

@@ -12,76 +12,14 @@ import type {
 } from "../types.js";
 import type { AnalysisContext } from "../analysis-context.js";
 import { boundedFetchJson } from "../external-fetch.js";
+import { isBinaryFileExtension } from "./binary-extensions.js";
 
 const MAX_FINDINGS = 50; // keep the brief bounded after evaluating every changed binary candidate
 const MAX_PATH_SIZE_LOOKUPS = 50; // fallback Contents API calls when a recursive tree is truncated
 const THRESHOLD_BYTES = 100 * 1024; // flag a newly-added blob >= 100 KB, or growth >= 100 KB
 const GITHUB_API = "https://github.com/ghapi";
 const GITHUB_API_VERSION = "2022-11-28";
 
-// Extensions that are genuinely binary (text formats like .svg/.json are excluded — their bytes are in the diff).
-const BINARY_EXTS = new Set([
-  "png",
-  "jpg",
-  "jpeg",
-  "gif",
-  "bmp",
-  "tiff",
-  "tif",
-  "ico",
-  "webp",
-  "avif",
-  "heic",
-  "heif",
-  "woff",
-  "woff2",
-  "ttf",
-  "otf",
-  "eot",
-  "mp4",
-  "mov",
-  "avi",
-  "webm",
-  "mkv",
-  "mp3",
-  "wav",
-  "flac",
-  "ogg",
-  "zip",
-  "tar",
-  "gz",
-  "tgz",
-  "bz2",
-  "7z",
-  "rar",
-  "xz",
-  "zst",
-  "pdf",
-  "psd",
-  "ai",
-  "sketch",
-  "fig",
-  "xcf",
-  "exe",
-  "dll",
-  "so",
-  "dylib",
-  "bin",
-  "dat",
-  "wasm",
-  "node",
-  "jar",
-  "class",
-  // Serialized ML model / checkpoint weight formats — routinely hundreds of MB to multi-GB, the heaviest
-  // binary blobs a PR can commit, and their bytes never appear in the textual diff.
-  "safetensors",
-  "gguf",
-  "onnx",
-  "pt",
-  "pth",
-  "ckpt",
-]);
-
 interface ScanOptions {
   signal?: AbortSignal;
   analysis?: Pick<AnalysisContext, "fetchJson">;
@@ -97,7 +35,7 @@ const SHA_RE = /^[0-9a-fA-F]{7,64}$/;
  *  like .svg/.json are deliberately excluded — their bytes are already in the textual diff. Pure. */
 export function isBinaryAsset(path: string): boolean {
   const dot = path.lastIndexOf(".");
-  return dot >= 0 && BINARY_EXTS.has(path.slice(dot + 1).toLowerCase());
+  return dot >= 0 && isBinaryFileExtension(path.slice(dot + 1));
 }
 
 type EnrichFile = NonNullable<EnrichRequest["files"]>[number];

review-enrichment/src/analyzers/binary-extensions.ts

@@ -0,0 +1,104 @@
+// Shared binary-file extension inventory for analyzers that classify opaque blobs by path extension.
+// asset-weight.ts (size bloat) and provenance.ts (unauditable committed artifacts) must stay in parity —
+// a single source list prevents one analyzer from drifting and missing formats the other already flags.
+
+/** Lowercase extensions for genuinely binary assets. Text formats like .svg/.json are excluded. */
+export const BINARY_FILE_EXTENSIONS = [
+  // Images
+  "png",
+  "jpg",
+  "jpeg",
+  "gif",
+  "bmp",
+  "tiff",
+  "tif",
+  "ico",
+  "webp",
+  "avif",
+  "heic",
+  "heif",
+  // Fonts
+  "woff",
+  "woff2",
+  "ttf",
+  "otf",
+  "eot",
+  // Media
+  "mp4",
+  "mov",
+  "avi",
+  "webm",
+  "mkv",
+  "mp3",
+  "wav",
+  "flac",
+  "ogg",
+  // Archives / compression
+  "zip",
+  "tar",
+  "gz",
+  "tgz",
+  "bz2",
+  "7z",
+  "rar",
+  "xz",
+  "zst",
+  "lz4",
+  "br",
+  // Documents / design
+  "pdf",
+  "psd",
+  "ai",
+  "sketch",
+  "fig",
+  "xcf",
+  // Native / compiled
+  "exe",
+  "dll",
+  "so",
+  "dylib",
+  "bin",
+  "dat",
+  "wasm",
+  "node",
+  "jar",
+  "class",
+  "pyc",
+  "pyo",
+  "pyd",
+  "o",
+  "a",
+  "war",
+  "ear",
+  // ML checkpoints
+  "safetensors",
+  "gguf",
+  "onnx",
+  "pt",
+  "pth",
+  "ckpt",
+  // Scientific / ML data artifacts
+  "h5",
+  "hdf5",
+  "pb",
+  "npy",
+  "npz",
+  "parquet",
+  "feather",
+  "arrow",
+  "orc",
+  "msgpack",
+] as const;
+
+const BINARY_EXT_SET = new Set<string>(BINARY_FILE_EXTENSIONS);
+
+/** True when `ext` (without a leading dot) is a known binary file extension. Case-insensitive. Pure. */
+export function isBinaryFileExtension(ext: string): boolean {
+  return BINARY_EXT_SET.has(ext.toLowerCase());
+}
+
+/** Extension-anchored, case-insensitive regex matching any shared binary extension at path end. Pure. */
+export const BINARY_EXT_RE = new RegExp(
+  `\\.(?:${BINARY_FILE_EXTENSIONS.join("|")})$`,
+  "i",
+);

review-enrichment/src/analyzers/provenance.ts

@@ -13,17 +13,14 @@ import type {
 import type { AnalysisContext } from "../analysis-context.js";
 import { extractDependencyChanges } from "./dependency-scan.js";
 import { boundedFetchJson } from "../external-fetch.js";
+import { BINARY_EXT_RE } from "./binary-extensions.js";
 
 const MAX_ATTESTATION_CHECKS = 20; // bound network round-trips
 const MAX_FINDINGS = 30; // keep the brief bounded
 
-// Compiled/non-source binary artifact extensions. `node` is a compiled Node native addon (matching the
-// binary set in asset-weight.ts) and `pyd` is a Windows Python extension DLL (the sibling of the pyc/pyo/so
-// entries) — both are unauditable prebuilt binaries a PR should not check in without source. ML checkpoint
-// formats (`safetensors`, `gguf`, `onnx`, `pt`, `pth`, `ckpt`) mirror asset-weight.ts — unauditable weight
-// blobs a PR should not commit without reproducible training source.
-const BINARY_EXT_RE =
-  /\.(?:exe|dll|so|dylib|bin|pyc|pyo|pyd|class|jar|war|ear|wasm|node|o|a|safetensors|gguf|onnx|pt|pth|ckpt)$/i;
+// Compiled/non-source binary artifact extensions. Shared with asset-weight.ts via binary-extensions.ts so
+// the size-bloat and provenance classifiers cannot drift. ML checkpoints and scientific data artifacts are
+// unauditable opaque blobs a PR should not commit without reproducible source.
 // Vendored / embedded third-party source trees. bower_components (Bower) and jspm_packages (JSPM) are
 // installed-dependency directories — the same vendored case as node_modules — so a committed tree under either
 // is a vendored artifact, not contributor source (mirrors src/signals/path-matchers.ts's vendored classifier).

review-enrichment/test/asset-weight.test.ts

@@ -41,6 +41,26 @@ test("isBinaryAsset flags genuine binary extensions and ignores text/case", () =
   ]) {
     assert.equal(isBinaryAsset(p), true, p);
   }
+  // Scientific / ML data artifacts and additional compression formats from the shared inventory.
+  for (const p of [
+    "data/train.h5",
+    "data/features.hdf5",
+    "models/saved_model.pb",
+    "data/embeddings.npy",
+    "data/batch.npz",
+    "warehouse/events.parquet",
+    "warehouse/snapshot.feather",
+    "lake/part-000.arrow",
+    "lake/part-000.orc",
+    "wire/msg.msgpack",
+    "cache/snapshot.lz4",
+    "dist/bundle.br",
+    "data/TRAIN.H5",
+  ]) {
+    assert.equal(isBinaryAsset(p), true, p);
+  }
+  assert.equal(isBinaryAsset("src/parquet.ts"), false);
+  assert.equal(isBinaryAsset("lib/npy_utils.py"), false);
   // Extension match is case-insensitive.
   assert.equal(isBinaryAsset("assets/HERO.PNG"), true);
   // Text formats whose bytes are already in the diff are NOT binary assets.

review-enrichment/test/binary-extensions.test.ts

@@ -0,0 +1,77 @@
+// Units for the shared binary extension inventory. Keeps asset-weight and provenance parity testable in one place.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  BINARY_EXT_RE,
+  BINARY_FILE_EXTENSIONS,
+  isBinaryFileExtension,
+} from "../dist/analyzers/binary-extensions.js";
+
+test("isBinaryFileExtension and BINARY_EXT_RE agree on known binary paths", () => {
+  for (const path of [
+    "assets/logo.png",
+    "cache/model.zst",
+    "dist/bundle.tar.br",
+    "snapshots/data.lz4",
+    "models/weights.safetensors",
+    "models/llama.gguf",
+    "data/train.h5",
+    "data/features.hdf5",
+    "models/saved_model.pb",
+    "data/embeddings.npy",
+    "data/batch.npz",
+    "warehouse/events.parquet",
+    "warehouse/snapshot.feather",
+    "lake/part-000.arrow",
+    "lake/part-000.orc",
+    "wire/msg.msgpack",
+    "native/mod.pyd",
+    "build/Release/addon.node",
+  ]) {
+    const ext = path.slice(path.lastIndexOf(".") + 1);
+    assert.equal(isBinaryFileExtension(ext), true, path);
+    assert.equal(BINARY_EXT_RE.test(path), true, path);
+  }
+});
+
+test("isBinaryFileExtension is case-insensitive", () => {
+  assert.equal(isBinaryFileExtension("PNG"), true);
+  assert.equal(isBinaryFileExtension("Parquet"), true);
+  assert.equal(BINARY_EXT_RE.test("data/TRAIN.H5"), true);
+});
+
+test("isBinaryFileExtension rejects text and extensionless paths", () => {
+  for (const path of [
+    "src/index.ts",
+    "icons/logo.svg",
+    "data/config.json",
+    "Makefile",
+    "src/parquet.ts",
+    "lib/npy_utils.py",
+  ]) {
+    const dot = path.lastIndexOf(".");
+    const ext = dot >= 0 ? path.slice(dot + 1) : "";
+    if (ext) assert.equal(isBinaryFileExtension(ext), false, path);
+    assert.equal(BINARY_EXT_RE.test(path), false, path);
+  }
+});
+
+test("BINARY_FILE_EXTENSIONS includes ML checkpoint and scientific data formats", () => {
+  for (const ext of [
+    "safetensors",
+    "gguf",
+    "onnx",
+    "h5",
+    "hdf5",
+    "parquet",
+    "feather",
+    "arrow",
+    "orc",
+    "msgpack",
+    "lz4",
+    "br",
+  ]) {
+    assert.ok(BINARY_FILE_EXTENSIONS.includes(ext), ext);
+  }
+});

review-enrichment/test/provenance.test.ts

@@ -51,3 +51,24 @@ test("classifyAddedFile flags committed ML checkpoint files as binary artifacts"
   assert.equal(classifyAddedFile("src/model.pt.ts"), null);
   assert.equal(classifyAddedFile("lib/onnx.ts"), null);
 });
+
+test("classifyAddedFile flags committed scientific data and columnar artifacts as binary", () => {
+  for (const path of [
+    "data/train.h5",
+    "data/features.hdf5",
+    "models/saved_model.pb",
+    "data/embeddings.npy",
+    "data/batch.npz",
+    "warehouse/events.parquet",
+    "warehouse/snapshot.feather",
+    "lake/part-000.arrow",
+    "lake/part-000.orc",
+    "wire/msg.msgpack",
+    "cache/snapshot.lz4",
+    "dist/bundle.br",
+  ]) {
+    assert.equal(classifyAddedFile(path), "binary", path);
+  }
+  assert.equal(classifyAddedFile("src/parquet.ts"), null);
+  assert.equal(classifyAddedFile("lib/npy_utils.py"), null);
+});