From 3e78a288e2fad0ca53f63b23b2f0c0eaae054e08 Mon Sep 17 00:00:00 2001 From: bohdansolovie Date: Sun, 5 Jul 2026 00:51:40 +0200 Subject: [PATCH] feat(enrichment): unify binary extension inventory for asset-weight and provenance Extract shared binary-extensions module so asset-weight and provenance stay in sync, and add scientific/data formats (HDF5, NumPy, Parquet, Arrow, ORC, MessagePack, lz4, brotli) with dedicated parity tests. Co-authored-by: Cursor --- .../src/analyzers/asset-weight.ts | 66 +---------- .../src/analyzers/binary-extensions.ts | 104 ++++++++++++++++++ review-enrichment/src/analyzers/provenance.ts | 11 +- review-enrichment/test/asset-weight.test.ts | 20 ++++ .../test/binary-extensions.test.ts | 77 +++++++++++++ review-enrichment/test/provenance.test.ts | 21 ++++ 6 files changed, 228 insertions(+), 71 deletions(-) create mode 100644 review-enrichment/src/analyzers/binary-extensions.ts create mode 100644 review-enrichment/test/binary-extensions.test.ts diff --git a/review-enrichment/src/analyzers/asset-weight.ts b/review-enrichment/src/analyzers/asset-weight.ts index f142a3ebea..fd9a136606 100644 --- a/review-enrichment/src/analyzers/asset-weight.ts +++ b/review-enrichment/src/analyzers/asset-weight.ts @@ -12,6 +12,7 @@ import type { } from "../types.js"; import type { AnalysisContext } from "../analysis-context.js"; import { boundedFetchJson } from "../external-fetch.js"; +import { isBinaryFileExtension } from "./binary-extensions.js"; const MAX_FINDINGS = 50; // keep the brief bounded after evaluating every changed binary candidate const MAX_PATH_SIZE_LOOKUPS = 50; // fallback Contents API calls when a recursive tree is truncated @@ -19,69 +20,6 @@ const THRESHOLD_BYTES = 100 * 1024; // flag a newly-added blob >= 100 KB, or gro const GITHUB_API = "https://api.github.com"; const GITHUB_API_VERSION = "2022-11-28"; -// Extensions that are genuinely binary (text formats like .svg/.json are excluded — their bytes are in the diff). -const BINARY_EXTS = new Set([ - "png", - "jpg", - "jpeg", - "gif", - "bmp", - "tiff", - "tif", - "ico", - "webp", - "avif", - "heic", - "heif", - "woff", - "woff2", - "ttf", - "otf", - "eot", - "mp4", - "mov", - "avi", - "webm", - "mkv", - "mp3", - "wav", - "flac", - "ogg", - "zip", - "tar", - "gz", - "tgz", - "bz2", - "7z", - "rar", - "xz", - "zst", - "pdf", - "psd", - "ai", - "sketch", - "fig", - "xcf", - "exe", - "dll", - "so", - "dylib", - "bin", - "dat", - "wasm", - "node", - "jar", - "class", - // Serialized ML model / checkpoint weight formats — routinely hundreds of MB to multi-GB, the heaviest - // binary blobs a PR can commit, and their bytes never appear in the textual diff. - "safetensors", - "gguf", - "onnx", - "pt", - "pth", - "ckpt", -]); - interface ScanOptions { signal?: AbortSignal; analysis?: Pick; @@ -97,7 +35,7 @@ const SHA_RE = /^[0-9a-fA-F]{7,64}$/; * like .svg/.json are deliberately excluded — their bytes are already in the textual diff. Pure. */ export function isBinaryAsset(path: string): boolean { const dot = path.lastIndexOf("."); - return dot >= 0 && BINARY_EXTS.has(path.slice(dot + 1).toLowerCase()); + return dot >= 0 && isBinaryFileExtension(path.slice(dot + 1)); } type EnrichFile = NonNullable[number]; diff --git a/review-enrichment/src/analyzers/binary-extensions.ts b/review-enrichment/src/analyzers/binary-extensions.ts new file mode 100644 index 0000000000..5fbd052b4d --- /dev/null +++ b/review-enrichment/src/analyzers/binary-extensions.ts @@ -0,0 +1,104 @@ +// Shared binary-file extension inventory for analyzers that classify opaque blobs by path extension. +// asset-weight.ts (size bloat) and provenance.ts (unauditable committed artifacts) must stay in parity — +// a single source list prevents one analyzer from drifting and missing formats the other already flags. + +/** Lowercase extensions for genuinely binary assets. Text formats like .svg/.json are excluded. */ +export const BINARY_FILE_EXTENSIONS = [ + // Images + "png", + "jpg", + "jpeg", + "gif", + "bmp", + "tiff", + "tif", + "ico", + "webp", + "avif", + "heic", + "heif", + // Fonts + "woff", + "woff2", + "ttf", + "otf", + "eot", + // Media + "mp4", + "mov", + "avi", + "webm", + "mkv", + "mp3", + "wav", + "flac", + "ogg", + // Archives / compression + "zip", + "tar", + "gz", + "tgz", + "bz2", + "7z", + "rar", + "xz", + "zst", + "lz4", + "br", + // Documents / design + "pdf", + "psd", + "ai", + "sketch", + "fig", + "xcf", + // Native / compiled + "exe", + "dll", + "so", + "dylib", + "bin", + "dat", + "wasm", + "node", + "jar", + "class", + "pyc", + "pyo", + "pyd", + "o", + "a", + "war", + "ear", + // ML checkpoints + "safetensors", + "gguf", + "onnx", + "pt", + "pth", + "ckpt", + // Scientific / ML data artifacts + "h5", + "hdf5", + "pb", + "npy", + "npz", + "parquet", + "feather", + "arrow", + "orc", + "msgpack", +] as const; + +const BINARY_EXT_SET = new Set(BINARY_FILE_EXTENSIONS); + +/** True when `ext` (without a leading dot) is a known binary file extension. Case-insensitive. Pure. */ +export function isBinaryFileExtension(ext: string): boolean { + return BINARY_EXT_SET.has(ext.toLowerCase()); +} + +/** Extension-anchored, case-insensitive regex matching any shared binary extension at path end. Pure. */ +export const BINARY_EXT_RE = new RegExp( + `\\.(?:${BINARY_FILE_EXTENSIONS.join("|")})$`, + "i", +); diff --git a/review-enrichment/src/analyzers/provenance.ts b/review-enrichment/src/analyzers/provenance.ts index b20ebecb20..db8533dc57 100644 --- a/review-enrichment/src/analyzers/provenance.ts +++ b/review-enrichment/src/analyzers/provenance.ts @@ -13,17 +13,14 @@ import type { import type { AnalysisContext } from "../analysis-context.js"; import { extractDependencyChanges } from "./dependency-scan.js"; import { boundedFetchJson } from "../external-fetch.js"; +import { BINARY_EXT_RE } from "./binary-extensions.js"; const MAX_ATTESTATION_CHECKS = 20; // bound network round-trips const MAX_FINDINGS = 30; // keep the brief bounded -// Compiled/non-source binary artifact extensions. `node` is a compiled Node native addon (matching the -// binary set in asset-weight.ts) and `pyd` is a Windows Python extension DLL (the sibling of the pyc/pyo/so -// entries) — both are unauditable prebuilt binaries a PR should not check in without source. ML checkpoint -// formats (`safetensors`, `gguf`, `onnx`, `pt`, `pth`, `ckpt`) mirror asset-weight.ts — unauditable weight -// blobs a PR should not commit without reproducible training source. -const BINARY_EXT_RE = - /\.(?:exe|dll|so|dylib|bin|pyc|pyo|pyd|class|jar|war|ear|wasm|node|o|a|safetensors|gguf|onnx|pt|pth|ckpt)$/i; +// Compiled/non-source binary artifact extensions. Shared with asset-weight.ts via binary-extensions.ts so +// the size-bloat and provenance classifiers cannot drift. ML checkpoints and scientific data artifacts are +// unauditable opaque blobs a PR should not commit without reproducible source. // Vendored / embedded third-party source trees. bower_components (Bower) and jspm_packages (JSPM) are // installed-dependency directories — the same vendored case as node_modules — so a committed tree under either // is a vendored artifact, not contributor source (mirrors src/signals/path-matchers.ts's vendored classifier). diff --git a/review-enrichment/test/asset-weight.test.ts b/review-enrichment/test/asset-weight.test.ts index 01f9b6a069..30e1c6c0c6 100644 --- a/review-enrichment/test/asset-weight.test.ts +++ b/review-enrichment/test/asset-weight.test.ts @@ -41,6 +41,26 @@ test("isBinaryAsset flags genuine binary extensions and ignores text/case", () = ]) { assert.equal(isBinaryAsset(p), true, p); } + // Scientific / ML data artifacts and additional compression formats from the shared inventory. + for (const p of [ + "data/train.h5", + "data/features.hdf5", + "models/saved_model.pb", + "data/embeddings.npy", + "data/batch.npz", + "warehouse/events.parquet", + "warehouse/snapshot.feather", + "lake/part-000.arrow", + "lake/part-000.orc", + "wire/msg.msgpack", + "cache/snapshot.lz4", + "dist/bundle.br", + "data/TRAIN.H5", + ]) { + assert.equal(isBinaryAsset(p), true, p); + } + assert.equal(isBinaryAsset("src/parquet.ts"), false); + assert.equal(isBinaryAsset("lib/npy_utils.py"), false); // Extension match is case-insensitive. assert.equal(isBinaryAsset("assets/HERO.PNG"), true); // Text formats whose bytes are already in the diff are NOT binary assets. diff --git a/review-enrichment/test/binary-extensions.test.ts b/review-enrichment/test/binary-extensions.test.ts new file mode 100644 index 0000000000..2ae0395202 --- /dev/null +++ b/review-enrichment/test/binary-extensions.test.ts @@ -0,0 +1,77 @@ +// Units for the shared binary extension inventory. Keeps asset-weight and provenance parity testable in one place. +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + BINARY_EXT_RE, + BINARY_FILE_EXTENSIONS, + isBinaryFileExtension, +} from "../dist/analyzers/binary-extensions.js"; + +test("isBinaryFileExtension and BINARY_EXT_RE agree on known binary paths", () => { + for (const path of [ + "assets/logo.png", + "cache/model.zst", + "dist/bundle.tar.br", + "snapshots/data.lz4", + "models/weights.safetensors", + "models/llama.gguf", + "data/train.h5", + "data/features.hdf5", + "models/saved_model.pb", + "data/embeddings.npy", + "data/batch.npz", + "warehouse/events.parquet", + "warehouse/snapshot.feather", + "lake/part-000.arrow", + "lake/part-000.orc", + "wire/msg.msgpack", + "native/mod.pyd", + "build/Release/addon.node", + ]) { + const ext = path.slice(path.lastIndexOf(".") + 1); + assert.equal(isBinaryFileExtension(ext), true, path); + assert.equal(BINARY_EXT_RE.test(path), true, path); + } +}); + +test("isBinaryFileExtension is case-insensitive", () => { + assert.equal(isBinaryFileExtension("PNG"), true); + assert.equal(isBinaryFileExtension("Parquet"), true); + assert.equal(BINARY_EXT_RE.test("data/TRAIN.H5"), true); +}); + +test("isBinaryFileExtension rejects text and extensionless paths", () => { + for (const path of [ + "src/index.ts", + "icons/logo.svg", + "data/config.json", + "Makefile", + "src/parquet.ts", + "lib/npy_utils.py", + ]) { + const dot = path.lastIndexOf("."); + const ext = dot >= 0 ? path.slice(dot + 1) : ""; + if (ext) assert.equal(isBinaryFileExtension(ext), false, path); + assert.equal(BINARY_EXT_RE.test(path), false, path); + } +}); + +test("BINARY_FILE_EXTENSIONS includes ML checkpoint and scientific data formats", () => { + for (const ext of [ + "safetensors", + "gguf", + "onnx", + "h5", + "hdf5", + "parquet", + "feather", + "arrow", + "orc", + "msgpack", + "lz4", + "br", + ]) { + assert.ok(BINARY_FILE_EXTENSIONS.includes(ext), ext); + } +}); diff --git a/review-enrichment/test/provenance.test.ts b/review-enrichment/test/provenance.test.ts index 6c048d25c3..cb410586f9 100644 --- a/review-enrichment/test/provenance.test.ts +++ b/review-enrichment/test/provenance.test.ts @@ -51,3 +51,24 @@ test("classifyAddedFile flags committed ML checkpoint files as binary artifacts" assert.equal(classifyAddedFile("src/model.pt.ts"), null); assert.equal(classifyAddedFile("lib/onnx.ts"), null); }); + +test("classifyAddedFile flags committed scientific data and columnar artifacts as binary", () => { + for (const path of [ + "data/train.h5", + "data/features.hdf5", + "models/saved_model.pb", + "data/embeddings.npy", + "data/batch.npz", + "warehouse/events.parquet", + "warehouse/snapshot.feather", + "lake/part-000.arrow", + "lake/part-000.orc", + "wire/msg.msgpack", + "cache/snapshot.lz4", + "dist/bundle.br", + ]) { + assert.equal(classifyAddedFile(path), "binary", path); + } + assert.equal(classifyAddedFile("src/parquet.ts"), null); + assert.equal(classifyAddedFile("lib/npy_utils.py"), null); +});