feat(core)!: replace Buffer-based byte encoding with uint8array-extras

.changeset/loose-cows-love.md

@@ -0,0 +1,8 @@
+---
+"@ckb-ccc/core": minor
+---
+
+feat(core)!: replace `Buffer`-based byte encoding with `uint8array-extras`
+
+- Remove the legacy `ascii`, `binary`, `latin1`, `ucs2`, and `utf16le` encodings
+- Use consistent strict validation for implicit and explicit hex input

packages/core/package.json

@@ -63,9 +63,9 @@
     "@noble/hashes": "^2.2.0",
     "bech32": "^2.0.0",
     "bs58check": "^4.0.0",
-    "buffer": "^6.0.3",
     "ethers": "^6.17.0",
     "isomorphic-ws": "^5.0.0",
+    "uint8array-extras": "^1.5.0",
     "ws": "^8.21.0"
   },
   "packageManager": "pnpm@11.8.0",
@@ -79,6 +79,7 @@
     ],
     "base-x": "5.0.1",
     "bs58": "6.0.0",
-    "bs58check": "4.0.0"
+    "bs58check": "4.0.0",
+    "uint8array-extras": "1.5.0"
   }
 }

packages/core/src/bytes/advanced.ts

@@ -4,11 +4,6 @@
  */
 export type BytesFromEncoding =
   | "utf8" // UTF-8 encoding
-  | "utf16le" // UTF-16 Little Endian encoding
-  | "latin1" // Latin-1 (ISO-8859-1) encoding
   | "base64" // Base64 encoding
   | "base64url" // Base64 URL encoding
-  | "hex" // Hexadecimal encoding
-  | "ascii" // ASCII encoding
-  | "binary" // Binary encoding
-  | "ucs2"; // UCS-2 (alias of UTF-16LE) encoding
+  | "hex"; // Hexadecimal encoding

packages/core/src/bytes/index.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import { bytesFrom, bytesTo } from "./index.js";
+
+describe("bytes", () => {
+  it("encodes and decodes UTF-8", () => {
+    const value = "Hello, 世界";
+
+    expect(bytesTo(bytesFrom(value, "utf8"), "utf8")).toBe(value);
+  });
+
+  it("encodes and decodes Base64", () => {
+    const bytes = Uint8Array.from([0xfb, 0xff]);
+
+    expect(bytesTo(bytes, "base64")).toBe("+/8=");
+    expect(bytesFrom("+/8=", "base64")).toEqual(bytes);
+  });
+
+  it("encodes and decodes unpadded Base64URL", () => {
+    const bytes = Uint8Array.from([0xfb, 0xff]);
+
+    expect(bytesTo(bytes, "base64url")).toBe("-_8");
+    expect(bytesFrom("-_8", "base64url")).toEqual(bytes);
+    expect(bytesFrom("+/8=", "base64url")).toEqual(bytes);
+  });
+
+  it("preserves hexadecimal input behavior", () => {
+    expect(bytesFrom("abc")).toEqual(Uint8Array.from([0x0a, 0xbc]));
+    expect(bytesFrom("abc", "hex")).toEqual(Uint8Array.from([0x0a, 0xbc]));
+    expect(bytesFrom("0xabc", "hex")).toEqual(Uint8Array.from([0x0a, 0xbc]));
+    expect(bytesFrom("ABcd")).toEqual(Uint8Array.from([0xab, 0xcd]));
+    expect(() => bytesFrom("xyz")).toThrow();
+    expect(() => bytesFrom("abzz", "hex")).toThrow();
+  });
+});

packages/core/src/bytes/index.ts

@@ -1,5 +1,12 @@
-import { Buffer } from "buffer/index.js";
-import { BytesFromEncoding } from "./advanced.js";
+import {
+  base64ToUint8Array,
+  hexToUint8Array,
+  stringToUint8Array,
+  uint8ArrayToBase64,
+  uint8ArrayToHex,
+  uint8ArrayToString,
+} from "uint8array-extras";
+import type { BytesFromEncoding } from "./advanced.js";
 
 /**
  * @public
@@ -91,7 +98,18 @@ export function bytesConcat(...args: BytesLike[]): Bytes {
  */
 
 export function bytesTo(val: BytesLike, encoding: BytesFromEncoding): string {
-  return Buffer.from(bytesFrom(val)).toString(encoding);
+  const bytes = bytesFrom(val);
+
+  switch (encoding) {
+    case "utf8":
+      return uint8ArrayToString(bytes);
+    case "base64":
+      return uint8ArrayToBase64(bytes);
+    case "base64url":
+      return uint8ArrayToBase64(bytes, { urlSafe: true });
+    case "hex":
+      return uint8ArrayToHex(bytes);
+  }
 }
 
 /**
@@ -133,17 +151,13 @@ export function bytesFrom(
   }
 
   if (typeof bytes === "string") {
-    if (encoding !== undefined) {
-      return Buffer.from(bytes, encoding);
+    if (encoding !== undefined && encoding !== "hex") {
+      return bytesFromString(bytes, encoding);
     }
 
     const str = bytes.startsWith("0x") ? bytes.slice(2) : bytes;
     const paddedStr = str.length % 2 === 0 ? str : `0${str}`;
-    const data = Buffer.from(paddedStr, "hex");
-    if (data.length * 2 !== paddedStr.length) {
-      throw new Error(`Invalid bytes ${bytes}`);
-    }
-    return data;
+    return hexToUint8Array(paddedStr);
   }
 
   const bytesArr = Array.from(bytes);
@@ -153,6 +167,19 @@ export function bytesFrom(
   return new Uint8Array(bytes);
 }
 
+function bytesFromString(
+  value: string,
+  encoding: Exclude<BytesFromEncoding, "hex">,
+): Bytes {
+  switch (encoding) {
+    case "utf8":
+      return stringToUint8Array(value);
+    case "base64":
+    case "base64url":
+      return base64ToUint8Array(value);
+  }
+}
+
 /**
  * Compares two byte-like values for equality.
  * @public

packages/core/src/hex/index.test.ts

@@ -21,7 +21,7 @@ describe("hexFrom", () => {
   );
 
   test("throws for invalid hex string", () => {
-    expect(() => hexFrom("0xzz")).toThrow("Invalid bytes 0xzz");
+    expect(() => hexFrom("0xzz")).toThrow();
   });
 
   test("throws for invalid byte values", () => {
@@ -48,7 +48,7 @@ describe("bytesLen", () => {
   );
 
   test("throws for invalid hex string", () => {
-    expect(() => bytesLen("0xzz")).toThrow("Invalid bytes 0xzz");
+    expect(() => bytesLen("0xzz")).toThrow();
   });
 
   test("throws for invalid byte values", () => {

packages/core/tsdown.config.mts

@@ -25,6 +25,7 @@ const bundleDeps = [
   "bs58check",
   "bs58", // By bs58check
   "base-x", // By bs58 - bs58check
+  "uint8array-extras",
 ] as string[];
 
 export default defineConfig(

packages/playground/src/app/page.tsx

@@ -90,7 +90,7 @@ async function shareToNostr(
         ...sent
           .map((relay) => {
             console.log(relay);
-            const bytes = ccc.bytesFrom(relay, "ascii");
+            const bytes = ccc.bytesFrom(relay, "utf8");
             return [[1, bytes.length], bytes];
           })
           .flat(),
@@ -129,7 +129,7 @@ async function getFromNEvent(
       eventId = ccc.hexFrom(value).slice(2);
     }
     if (type === 1) {
-      const relay = ccc.bytesTo(value, "ascii");
+      const relay = ccc.bytesTo(value, "utf8");
       if (!relays.includes(relay)) {
         relays.push(relay);
       }