From bda1735d715838e4182d3356378a14b474cc1e16 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 19:31:16 +0900 Subject: [PATCH 01/13] add wasm implementation again --- benchmark/decode-string.ts | 31 +++- benchmark/encode-string.ts | 29 ++- benchmark/key-decoder.ts | 2 +- package.json | 1 + src/utils/utf8-wasm-binary.ts | 4 + src/utils/utf8-wasm.ts | 197 ++++++++++++++++++++ src/utils/utf8.ts | 49 ++++- test/utf8-wasm.test.ts | 130 +++++++++++++ wasm/.gitignore | 2 + wasm/README.md | 334 ++++++++++++++++++++++++++++++++++ wasm/build.sh | 25 +++ wasm/utf8.wat | 244 +++++++++++++++++++++++++ 12 files changed, 1040 insertions(+), 8 deletions(-) create mode 100644 src/utils/utf8-wasm-binary.ts create mode 100644 src/utils/utf8-wasm.ts create mode 100644 test/utf8-wasm.test.ts create mode 100644 wasm/.gitignore create mode 100644 wasm/README.md create mode 100755 wasm/build.sh create mode 100644 wasm/utf8.wat diff --git a/benchmark/decode-string.ts b/benchmark/decode-string.ts index a6ea1467..c02d2041 100644 --- a/benchmark/decode-string.ts +++ b/benchmark/decode-string.ts @@ -1,9 +1,27 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD, WASM_AVAILABLE } from "../src/utils/utf8.ts"; +import { getWasmError, utf8DecodeWasm } from "../src/utils/utf8-wasm.ts"; // @ts-ignore import Benchmark from "benchmark"; +// Show wasm status +console.log("=".repeat(60)); +console.log("WebAssembly Status:"); +console.log(` WASM_AVAILABLE: ${WASM_AVAILABLE}`); +if (WASM_AVAILABLE) { + console.log(" js-string-builtins: enabled"); +} else { + const error = getWasmError(); + console.log(` Error: ${error?.message || "unknown"}`); + if (error?.message?.includes("js-string") || error?.message?.includes("builtin")) { + console.log("\n js-string-builtins is enabled by default in Node.js 24+ (V8 13.6+)."); + console.log(" For older versions, run with:"); + console.log(" node --experimental-wasm-imported-strings node_modules/.bin/ts-node benchmark/decode-string.ts"); + } +} +console.log("=".repeat(60)); + for (const baseStr of ["A", "あ", "🌏"]) { const dataSet = [10, 100, 500, 1_000].map((n) => { return baseStr.repeat(n); @@ -24,11 +42,20 @@ for (const baseStr of ["A", "あ", "🌏"]) { } }); - suite.add("TextDecoder", () => { + suite.add("utf8DecodeTD (TextDecoder)", () => { if (utf8DecodeTD(bytes, 0, byteLength) !== str) { throw new Error("wrong result!"); } }); + + if (WASM_AVAILABLE) { + suite.add("utf8DecodeWasm", () => { + if (utf8DecodeWasm(bytes, 0, byteLength) !== str) { + throw new Error("wrong result!"); + } + }); + } + suite.on("cycle", (event: any) => { console.log(String(event.target)); }); diff --git a/benchmark/encode-string.ts b/benchmark/encode-string.ts index 3f6aac6c..ac7ed8dc 100644 --- a/benchmark/encode-string.ts +++ b/benchmark/encode-string.ts @@ -1,9 +1,27 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8EncodeTE, WASM_AVAILABLE } from "../src/utils/utf8.ts"; +import { getWasmError, utf8EncodeWasm } from "../src/utils/utf8-wasm.ts"; // @ts-ignore import Benchmark from "benchmark"; +// Show wasm status +console.log("=".repeat(60)); +console.log("WebAssembly Status:"); +console.log(` WASM_AVAILABLE: ${WASM_AVAILABLE}`); +if (WASM_AVAILABLE) { + console.log(" js-string-builtins: enabled"); +} else { + const error = getWasmError(); + console.log(` Error: ${error?.message || "unknown"}`); + if (error?.message?.includes("js-string") || error?.message?.includes("builtin")) { + console.log("\n js-string-builtins is enabled by default in Node.js 24+ (V8 13.6+)."); + console.log(" For older versions, run with:"); + console.log(" node --experimental-wasm-imported-strings node_modules/.bin/ts-node benchmark/encode-string.ts"); + } +} +console.log("=".repeat(60)); + for (const baseStr of ["A", "あ", "🌏"]) { const dataSet = [10, 30, 50, 100].map((n) => { return baseStr.repeat(n); @@ -21,9 +39,16 @@ for (const baseStr of ["A", "あ", "🌏"]) { utf8EncodeJs(str, buffer, 0); }); - suite.add("utf8DecodeTE", () => { + suite.add("utf8EncodeTE (TextEncoder)", () => { utf8EncodeTE(str, buffer, 0); }); + + if (WASM_AVAILABLE) { + suite.add("utf8EncodeWasm", () => { + utf8EncodeWasm(str, buffer, 0); + }); + } + suite.on("cycle", (event: any) => { console.log(String(event.target)); }); diff --git a/benchmark/key-decoder.ts b/benchmark/key-decoder.ts index 594bbab2..55ab4819 100644 --- a/benchmark/key-decoder.ts +++ b/benchmark/key-decoder.ts @@ -1,5 +1,5 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8DecodeJs } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8DecodeJs } from "../src/utils/utf8.ts"; // @ts-ignore import Benchmark from "benchmark"; diff --git a/package.json b/package.json index 67e4dd4a..37417809 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "prepublishOnly": "npm run test:dist", "clean": "rimraf build dist dist.*", "test": "mocha 'test/**/*.test.ts'", + "test:wasm": "MSGPACK_WASM=force node --experimental-wasm-imported-strings node_modules/.bin/mocha 'test/**/*.test.ts'", "test:dist": "npm run lint && npm run test && npm run test:deno", "test:cover": "npm run cover:clean && npx nyc --no-clean npm run 'test' && npm run cover:report", "test:node_with_strip_types": "node --experimental-strip-types test/deno_test.ts", diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts new file mode 100644 index 00000000..4ca1b240 --- /dev/null +++ b/src/utils/utf8-wasm-binary.ts @@ -0,0 +1,4 @@ +// Auto-generated by wasm/build.sh - do not edit manually +// Source: wasm/utf8.wat + +export const wasmBinary = "AGFzbQEAAAABHwVgAW8Bf2ACb38Bf2ABfwFkb2ACb28BZG9gAn9/AW8CawQOd2FzbTpqcy1zdHJpbmcGbGVuZ3RoAAAOd2FzbTpqcy1zdHJpbmcKY2hhckNvZGVBdAABDndhc206anMtc3RyaW5nDGZyb21DaGFyQ29kZQACDndhc206anMtc3RyaW5nBmNvbmNhdAADAwQDAAEEBQMBAAEHMAQGbWVtb3J5AgAJdXRmOENvdW50AAQKdXRmOEVuY29kZQAFCnV0ZjhEZWNvZGUABgqXBgN1AQR/IAAQACECAkADQCABIAJPDQEgACABEAEhBCAEQYABSQRAIANBAWohAwUgBEGAEEkEQCADQQJqIQMFIARBgLADTyAEQf+3A01xBEAgA0EEaiEDIAFBAWohAQUgA0EDaiEDCwsLIAFBAWohAQwACwALIAMLvQIBBX8gABAAIQMgASEEAkADQCACIANPDQEgACACEAEhBSAFQYABSQRAIAQgBToAACAEQQFqIQQFIAVBgBBJBEAgBCAFQQZ2QcABcjoAACAEQQFqIAVBP3FBgAFyOgAAIARBAmohBAUgBUGAsANPIAVB/7cDTXEEQCACQQFqIQIgACACEAEhBiAFQYCwA2tBCnQgBkGAuANrakGAgARqIQUgBCAFQRJ2QfABcjoAACAEQQFqIAVBDHZBP3FBgAFyOgAAIARBAmogBUEGdkE/cUGAAXI6AAAgBEEDaiAFQT9xQYABcjoAACAEQQRqIQQFIAQgBUEMdkHgAXI6AAAgBEEBaiAFQQZ2QT9xQYABcjoAACAEQQJqIAVBP3FBgAFyOgAAIARBA2ohBAsLCyACQQFqIQIMAAsACyAEIAFrC98CAgd/AW8gACECIAAgAWohA0EAEAIhCQJAA0AgAiADTw0BIAItAAAhBCAEQYABcUUEQCAJIAQQAhADIQkgAkEBaiECDAELIARB4AFxQcABRgRAIAJBAWotAAAhBSAEQR9xQQZ0IAVBP3FyIQggCSAIEAIQAyEJIAJBAmohAgwBCyAEQfABcUHgAUYEQCACQQFqLQAAIQUgAkECai0AACEGIARBD3FBDHQgBUE/cUEGdHIgBkE/cXIhCCAJIAgQAhADIQkgAkEDaiECDAELIARB+AFxQfABRgRAIAJBAWotAAAhBSACQQJqLQAAIQYgAkEDai0AACEHIARBB3FBEnQgBUE/cUEMdHIgBkE/cUEGdHIgB0E/cXIhCCAIQYCABGshCCAJIAhBCnZBgLADchACEAMhCSAJIAhB/wdxQYC4A3IQAhADIQkgAkEEaiECDAELIAJBAWohAgwACwALIAkL"; diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts new file mode 100644 index 00000000..5fa9976f --- /dev/null +++ b/src/utils/utf8-wasm.ts @@ -0,0 +1,197 @@ +/** + * WebAssembly-based UTF-8 string processing using js-string-builtins. + * + * Environment variables: + * - MSGPACK_WASM=force: Force wasm mode, throw error if wasm fails to load + * - MSGPACK_WASM=never: Disable wasm, always use pure JS + * + * Three-tier fallback: + * 1. Native js-string-builtins (Chrome 130+, Firefox 134+) + * 2. Wasm + polyfill (older browsers with WebAssembly) + * 3. Pure JS (no WebAssembly support) + */ + +import { wasmBinary } from "./utf8-wasm-binary.ts"; + +// Check environment variable for wasm mode +declare const process: { env?: Record } | undefined; + +function getWasmMode(): "force" | "never" | "auto" { + try { + if (process?.env) { + const mode = process.env["MSGPACK_WASM"]; + if (mode) { + switch (mode.toLowerCase()) { + case "force": + return "force"; + case "never": + return "never"; + default: + return "auto"; + } + } + } + } catch { + // process may not be defined in browser + } + return "auto"; +} + +const WASM_MODE = getWasmMode(); + +interface WasmExports { + memory: WebAssembly.Memory; + utf8Count(str: string): number; + utf8Encode(str: string, offset: number): number; + utf8Decode(offset: number, length: number): string; +} + +let wasmInstance: WasmExports | null = null; +let wasmInitError: Error | null = null; + +function base64ToBytes(base64: string): Uint8Array { + if (typeof atob === "function") { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; + } + // Node.js fallback + return new Uint8Array(Buffer.from(base64, "base64")); +} + +// Polyfill for js-string-builtins (used when native builtins unavailable) +const jsStringPolyfill = { + // eslint-disable-next-line @typescript-eslint/naming-convention + "wasm:js-string": { + length: (s: string) => s.length, + charCodeAt: (s: string, i: number) => s.charCodeAt(i), + fromCharCode: (code: number) => String.fromCharCode(code), + concat: (a: string, b: string) => a + b, + }, +}; + +function tryInitWasm(): void { + if (wasmInstance !== null || wasmInitError !== null) { + return; // Already initialized or failed + } + + if (WASM_MODE === "never") { + wasmInitError = new Error("MSGPACK_WASM=never: wasm disabled"); + return; + } + + try { + if (typeof WebAssembly === "undefined") { + throw new Error("WebAssembly not supported"); + } + + const bytes = base64ToBytes(wasmBinary); + + // Try with builtins option (native support) + // If builtins not supported, option is ignored and polyfill is used + + + const module: WebAssembly.Module = new (WebAssembly.Module as any)(bytes, { builtins: ["js-string"] }); + + + const instance = new (WebAssembly.Instance)(module, jsStringPolyfill); + wasmInstance = instance.exports as unknown as WasmExports; + } catch (e) { + wasmInitError = e instanceof Error ? e : new Error(String(e)); + + if (WASM_MODE === "force") { + throw new Error(`MSGPACK_WASM=force but wasm failed to load: ${wasmInitError.message}`); + } + } +} + +// Initialize on module load +tryInitWasm(); + +/** + * Whether wasm is available and initialized. + */ +// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition +export const WASM_AVAILABLE = (wasmInstance !== null); + +/** + * Get the wasm initialization error, if any. + */ +export function getWasmError(): Error | null { + return wasmInitError; +} + +/** + * Get the raw wasm exports for advanced usage. + */ +export function getWasmExports(): WasmExports | null { + return wasmInstance; +} + +/** + * Count UTF-8 byte length of a string. + */ +export function utf8CountWasm(str: string): number { + if (wasmInstance === null) { + throw new Error("wasm not initialized"); + } + return wasmInstance.utf8Count(str); +} + +/** + * Encode string to UTF-8 bytes in the provided buffer. + * Returns the number of bytes written. + */ +export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: number): number { + if (wasmInstance === null) { + throw new Error("wasm not initialized"); + } + + // Ensure wasm memory is large enough + const byteLength = wasmInstance.utf8Count(str); + const requiredPages = Math.ceil((outputOffset + byteLength) / 65536); + const currentPages = wasmInstance.memory.buffer.byteLength / 65536; + + if (requiredPages > currentPages) { + wasmInstance.memory.grow(requiredPages - currentPages); + } + + // Encode to wasm memory + const bytesWritten = wasmInstance.utf8Encode(str, 0); + + // Copy from wasm memory to output buffer + const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, bytesWritten); + output.set(wasmBytes, outputOffset); + + return bytesWritten; +} + +/** + * Decode UTF-8 bytes to string. + */ +export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { + if (wasmInstance === null) { + throw new Error("wasm not initialized"); + } + + // Ensure wasm memory is large enough + const requiredPages = Math.ceil(byteLength / 65536); + const currentPages = wasmInstance.memory.buffer.byteLength / 65536; + + if (requiredPages > currentPages) { + wasmInstance.memory.grow(requiredPages - currentPages); + } + + // Copy bytes to wasm memory + const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, byteLength); + wasmBytes.set(bytes.subarray(inputOffset, inputOffset + byteLength)); + + // Decode from wasm memory + const result = wasmInstance.utf8Decode(0, byteLength); + + // Remove leading NUL character (artifact of wasm implementation) + return result.length > 0 && result.charCodeAt(0) === 0 ? result.slice(1) : result; +} diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts index 1494f70a..bb911e3a 100644 --- a/src/utils/utf8.ts +++ b/src/utils/utf8.ts @@ -1,4 +1,8 @@ -export function utf8Count(str: string): number { +import { WASM_AVAILABLE, utf8CountWasm, utf8EncodeWasm, utf8DecodeWasm } from "./utf8-wasm.ts"; + +export { WASM_AVAILABLE }; + +export function utf8CountJs(str: string): number { const strLength = str.length; let byteLength = 0; @@ -38,6 +42,8 @@ export function utf8Count(str: string): number { return byteLength; } +export const utf8Count: (str: string) => number = WASM_AVAILABLE ? utf8CountWasm : utf8CountJs; + export function utf8EncodeJs(str: string, output: Uint8Array, outputOffset: number): void { const strLength = str.length; let offset = outputOffset; @@ -98,7 +104,23 @@ export function utf8EncodeTE(str: string, output: Uint8Array, outputOffset: numb sharedTextEncoder.encodeInto(str, output.subarray(outputOffset)); } -export function utf8Encode(str: string, output: Uint8Array, outputOffset: number): void { +// Wasm threshold: use wasm for medium strings, TextEncoder for large strings +// These thresholds should be determined by benchmarking. +// Run `npx ts-node benchmark/encode-string.ts` for details. +const WASM_ENCODE_MAX = 1000; + +function utf8EncodeWithWasm(str: string, output: Uint8Array, outputOffset: number): void { + const len = str.length; + if (len > WASM_ENCODE_MAX) { + utf8EncodeTE(str, output, outputOffset); + } else if (len > TEXT_ENCODER_THRESHOLD) { + utf8EncodeWasm(str, output, outputOffset); + } else { + utf8EncodeJs(str, output, outputOffset); + } +} + +function utf8EncodeNoWasm(str: string, output: Uint8Array, outputOffset: number): void { if (str.length > TEXT_ENCODER_THRESHOLD) { utf8EncodeTE(str, output, outputOffset); } else { @@ -106,6 +128,10 @@ export function utf8Encode(str: string, output: Uint8Array, outputOffset: number } } +export const utf8Encode: (str: string, output: Uint8Array, outputOffset: number) => void = WASM_AVAILABLE + ? utf8EncodeWithWasm + : utf8EncodeNoWasm; + const CHUNK_SIZE = 0x1_000; export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: number): string { @@ -168,10 +194,27 @@ export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: return sharedTextDecoder.decode(stringBytes); } -export function utf8Decode(bytes: Uint8Array, inputOffset: number, byteLength: number): string { +// Wasm decode threshold: use wasm for medium strings, TextDecoder for large strings +const WASM_DECODE_MAX = 1000; + +function utf8DecodeWithWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { + if (byteLength > WASM_DECODE_MAX) { + return utf8DecodeTD(bytes, inputOffset, byteLength); + } else if (byteLength > TEXT_DECODER_THRESHOLD) { + return utf8DecodeWasm(bytes, inputOffset, byteLength); + } else { + return utf8DecodeJs(bytes, inputOffset, byteLength); + } +} + +function utf8DecodeNoWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { if (byteLength > TEXT_DECODER_THRESHOLD) { return utf8DecodeTD(bytes, inputOffset, byteLength); } else { return utf8DecodeJs(bytes, inputOffset, byteLength); } } + +export const utf8Decode: (bytes: Uint8Array, inputOffset: number, byteLength: number) => string = WASM_AVAILABLE + ? utf8DecodeWithWasm + : utf8DecodeNoWasm; diff --git a/test/utf8-wasm.test.ts b/test/utf8-wasm.test.ts new file mode 100644 index 00000000..4ba7355c --- /dev/null +++ b/test/utf8-wasm.test.ts @@ -0,0 +1,130 @@ +import assert from "assert"; +import { WASM_AVAILABLE, getWasmError, getWasmExports } from "../src/utils/utf8-wasm.ts"; +import { utf8Count, utf8CountJs, utf8Encode, utf8EncodeJs, utf8Decode, utf8DecodeJs } from "../src/utils/utf8.ts"; + +describe("utf8-wasm", () => { + describe("initialization", () => { + it("reports WASM_AVAILABLE status", () => { + // In Node.js without the flag, wasm should fail to load + // but we should get a clear error message + console.log("WASM_AVAILABLE:", WASM_AVAILABLE); + console.log("WASM error:", getWasmError()?.message); + + // Just verify the exports work + assert.strictEqual(typeof WASM_AVAILABLE, "boolean"); + }); + + it("getWasmExports returns null or valid exports", () => { + const exports = getWasmExports(); + if (WASM_AVAILABLE) { + assert.ok(exports !== null); + assert.ok(typeof exports!.utf8Count === "function"); + assert.ok(typeof exports!.utf8Encode === "function"); + assert.ok(typeof exports!.utf8Decode === "function"); + assert.ok(exports!.memory instanceof WebAssembly.Memory); + } else { + assert.strictEqual(exports, null); + } + }); + }); + + describe("utf8Count", () => { + const testCases = [ + { input: "", expected: 0, description: "empty string" }, + { input: "hello", expected: 5, description: "ASCII" }, + { input: "こんにけは", expected: 15, description: "Japanese hiragana (3 bytes each)" }, + { input: "πŸŽ‰", expected: 4, description: "emoji (4 bytes)" }, + { input: "helloπŸŽ‰world", expected: 14, description: "mixed ASCII and emoji" }, + { input: "Ξ©", expected: 2, description: "Greek omega (2 bytes)" }, + { input: "€", expected: 3, description: "Euro sign (3 bytes)" }, + { input: "π„ž", expected: 4, description: "Musical G clef (4 bytes, surrogate pair)" }, + ]; + + for (const { input, expected, description } of testCases) { + it(`counts ${description}: "${input}" = ${expected} bytes`, () => { + const jsResult = utf8CountJs(input); + const result = utf8Count(input); + + assert.strictEqual(jsResult, expected, `JS implementation failed for "${input}"`); + assert.strictEqual(result, expected, `utf8Count failed for "${input}"`); + }); + } + }); + + describe("utf8Encode", () => { + const testCases = [ + { input: "hello", description: "ASCII" }, + { input: "こんにけは", description: "Japanese" }, + { input: "πŸŽ‰πŸŽŠπŸŽ", description: "emojis" }, + { input: "helloπŸŽ‰world", description: "mixed" }, + { input: "Ξ©β‚¬π„ž", description: "multi-byte chars" }, + { input: "a".repeat(100), description: "100 ASCII chars" }, + { input: "ζ—₯".repeat(100), description: "100 Japanese chars" }, + ]; + + for (const { input, description } of testCases) { + it(`encodes ${description}`, () => { + const byteLength = utf8Count(input); + const buffer1 = new Uint8Array(byteLength); + const buffer2 = new Uint8Array(byteLength); + + utf8EncodeJs(input, buffer1, 0); + utf8Encode(input, buffer2, 0); + + // Compare with TextEncoder as ground truth + const expected = new TextEncoder().encode(input); + assert.deepStrictEqual(buffer1, expected, `JS encode failed for "${description}"`); + assert.deepStrictEqual(buffer2, expected, `utf8Encode failed for "${description}"`); + }); + } + }); + + describe("utf8Decode", () => { + const testCases = [ + { input: "hello", description: "ASCII" }, + { input: "こんにけは", description: "Japanese" }, + { input: "πŸŽ‰πŸŽŠπŸŽ", description: "emojis" }, + { input: "helloπŸŽ‰world", description: "mixed" }, + { input: "Ξ©β‚¬π„ž", description: "multi-byte chars" }, + { input: "a".repeat(100), description: "100 ASCII chars" }, + { input: "ζ—₯".repeat(100), description: "100 Japanese chars" }, + ]; + + for (const { input, description } of testCases) { + it(`decodes ${description}`, () => { + const bytes = new TextEncoder().encode(input); + + const jsResult = utf8DecodeJs(bytes, 0, bytes.length); + const result = utf8Decode(bytes, 0, bytes.length); + + assert.strictEqual(jsResult, input, `JS decode failed for "${description}"`); + assert.strictEqual(result, input, `utf8Decode failed for "${description}"`); + }); + } + }); + + describe("round-trip", () => { + const testStrings = [ + "", + "hello", + "Hello, δΈ–η•Œ! 🌍", + "The quick brown fox jumps over the lazy dog", + "ζ—₯本θͺžγƒ†γ‚Ήγƒˆ", + "Emoji: πŸ˜€πŸŽ‰πŸš€πŸ’»πŸ”₯", + "\u0000\u0001\u0002", // control characters + "Tab:\tNewline:\n", + "Mixed: ASCII, Ελληνικά, ζ—₯本θͺž, Ψ§Ω„ΨΉΨ±Ψ¨ΩŠΨ©, 🎌", + ]; + + for (const str of testStrings) { + it(`round-trips: "${str.slice(0, 30)}${str.length > 30 ? "..." : ""}"`, () => { + const byteLength = utf8Count(str); + const buffer = new Uint8Array(byteLength); + utf8Encode(str, buffer, 0); + const decoded = utf8Decode(buffer, 0, byteLength); + + assert.strictEqual(decoded, str); + }); + } + }); +}); diff --git a/wasm/.gitignore b/wasm/.gitignore new file mode 100644 index 00000000..44ddaba8 --- /dev/null +++ b/wasm/.gitignore @@ -0,0 +1,2 @@ +# Generated wasm binary - rebuild with ./build.sh +*.wasm diff --git a/wasm/README.md b/wasm/README.md new file mode 100644 index 00000000..be8935e9 --- /dev/null +++ b/wasm/README.md @@ -0,0 +1,334 @@ +# WebAssembly String Processing Plan + +## Background + +### Previous Attempt (2019-2020) + +- **PR #26**: Introduced AssemblyScript-based UTF-8 encode/decode +- **PR #95**: Removed it because "Wasm for UTF-8 encode/decode is not much faster than pureJS" + +The main issues were: +1. JS-to-Wasm call overhead negated encoding gains +2. String copying between JS and Wasm memory was expensive +3. Maintenance burden wasn't justified by performance gains + +### What Changed in 2025 + +**js-string-builtins** (WebAssembly 3.0, September 2025) fundamentally changes the equation: + +- Direct import of JS string operations (`length`, `charCodeAt`, `substring`, etc.) from `wasm:js-string` +- No glue code overhead - operations can be inlined by the engine +- No memory copying at boundaries when consuming JS strings +- Strings stay in JS representation (UTF-16) - no UTF-8/UTF-16 conversion + +Browser/runtime support: +- Chrome 131+ (enabled by default) +- Firefox 134+ +- Safari: TBD (expressed openness) +- Node.js 24+ (V8 13.6+, enabled by default) +- Node.js 22-23: `--experimental-wasm-imported-strings` flag required + +## Proposal: Hand-written WAT + +### Why WAT over Rust/wasm-bindgen + +| Aspect | Hand-written WAT | Rust + wasm-bindgen | +|--------|------------------|---------------------| +| Overhead | Zero - direct builtins | Glue code overhead | +| Binary size | Minimal (~1-2KB) | Larger (~10KB+) | +| Dependencies | None (just wat2wasm) | Rust toolchain, wasm-pack | +| Complexity | Simple for small scope | Overkill for 3 functions | +| js-string-builtins | Direct imports | Indirect, still evolving | +| Contributor barrier | Low (WAT is simple) | Higher (Rust knowledge) | + +For our limited scope (UTF-8 encode/decode), hand-written WAT is ideal. + +### What to Implement in Wasm + +1. **UTF-8 byte length counting** (`utf8Count`) + - Iterate string via `charCodeAt`, calculate byte length + +2. **UTF-8 encoding** (`utf8Encode`) + - Read chars via `charCodeAt`, write UTF-8 bytes to linear memory + +3. **UTF-8 decoding** (`utf8Decode`) + - Read UTF-8 bytes from memory, build string via `fromCharCode`/`fromCodePoint` + +### Available js-string-builtins + +From `wasm:js-string`: +- `length` - get string length +- `charCodeAt` - get UTF-16 code unit at index +- `codePointAt` - get Unicode code point at index (handles surrogates) +- `fromCharCode` - create single-char string from code unit +- `fromCodePoint` - create single-char string from code point +- `concat` - concatenate strings +- `substring` - extract substring +- `equals` - compare strings + +## Implementation Plan + +### Phase 1: Project Setup + +``` +msgpack-javascript/ +β”œβ”€β”€ wasm/ +β”‚ β”œβ”€β”€ utf8.wat # hand-written WAT source +β”‚ └── build.sh # wat2wasm + base64 generation +β”œβ”€β”€ src/ +β”‚ └── utils/ +β”‚ β”œβ”€β”€ utf8.ts # existing pure JS +β”‚ β”œβ”€β”€ utf8-wasm.ts # wasm loader + integration +β”‚ └── utf8-wasm-binary.ts # auto-generated base64 wasm +``` + +### Phase 2: WAT Implementation + +```wat +;; wasm/utf8.wat +(module + ;; Import js-string builtins + ;; Note: string parameters use externref, string returns use (ref extern) + (import "wasm:js-string" "length" + (func $str_length (param externref) (result i32))) + (import "wasm:js-string" "charCodeAt" + (func $str_charCodeAt (param externref i32) (result i32))) + (import "wasm:js-string" "fromCharCode" + (func $str_fromCharCode (param i32) (result (ref extern)))) + (import "wasm:js-string" "concat" + (func $str_concat (param externref externref) (result (ref extern)))) + + ;; Linear memory for UTF-8 bytes (exported for JS access) + (memory (export "memory") 1) + + ;; Count UTF-8 byte length of a JS string + (func (export "utf8Count") (param $str externref) (result i32) + (local $i i32) + (local $len i32) + (local $byteLen i32) + (local $code i32) + + (local.set $len (call $str_length (local.get $str))) + + (block $break + (loop $continue + (br_if $break (i32.ge_u (local.get $i) (local.get $len))) + + (local.set $code + (call $str_charCodeAt (local.get $str) (local.get $i))) + + ;; Count bytes based on code point range + (if (i32.lt_u (local.get $code) (i32.const 0x80)) + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 1)))) + (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 2)))) + (else (if (i32.and + (i32.ge_u (local.get $code) (i32.const 0xD800)) + (i32.le_u (local.get $code) (i32.const 0xDBFF))) + ;; High surrogate - 4 bytes total, skip low surrogate + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 4))) + (local.set $i (i32.add (local.get $i) (i32.const 1)))) + (else + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 3))))))))) + + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $continue))) + + (local.get $byteLen)) + + ;; Encode JS string to UTF-8 bytes at offset, returns bytes written + (func (export "utf8Encode") (param $str externref) (param $offset i32) (result i32) + ;; Similar loop: charCodeAt -> encode -> store to memory + (local $i i32) + (local $len i32) + (local $pos i32) + (local $code i32) + + (local.set $len (call $str_length (local.get $str))) + (local.set $pos (local.get $offset)) + + (block $break + (loop $continue + (br_if $break (i32.ge_u (local.get $i) (local.get $len))) + + (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) + + ;; 1-byte (ASCII) + (if (i32.lt_u (local.get $code) (i32.const 0x80)) + (then + (i32.store8 (local.get $pos) (local.get $code)) + (local.set $pos (i32.add (local.get $pos) (i32.const 1)))) + (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) + ;; 2-byte + (then + (i32.store8 (local.get $pos) + (i32.or (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0xC0))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) + (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) + ;; 3-byte or 4-byte (surrogate pair) + (else + ;; TODO: handle surrogates for 4-byte + (i32.store8 (local.get $pos) + (i32.or (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0xE0))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) + (i32.store8 (i32.add (local.get $pos) (i32.const 2)) + (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) + (local.set $pos (i32.add (local.get $pos) (i32.const 3))))))) + + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $continue))) + + (i32.sub (local.get $pos) (local.get $offset))) + + ;; Decode UTF-8 bytes from memory to JS string + (func (export "utf8Decode") (param $offset i32) (param $length i32) (result externref) + ;; Build string by reading bytes, decoding, calling fromCharCode + concat + ;; ... implementation + (call $str_fromCharCode (i32.const 0))) ;; placeholder +) +``` + +### Phase 3: Build Script + +```bash +#!/bin/bash +# wasm/build.sh +# Requires: binaryen (brew install binaryen) + +wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc + +# Generate base64-encoded TypeScript module +echo "// Auto-generated - do not edit" > ../src/utils/utf8-wasm-binary.ts +echo "export const wasmBinary = \"$(base64 -i utf8.wasm)\";" >> ../src/utils/utf8-wasm-binary.ts +``` + +### Phase 4: TypeScript Integration + +```typescript +// src/utils/utf8-wasm-binary.ts (auto-generated) +export const wasmBinary = "AGFzbQEAAAA..."; // base64-encoded wasm + +// src/utils/utf8-wasm.ts +import { utf8Count as utf8CountJs } from "./utf8.js"; +import { wasmBinary } from "./utf8-wasm-binary.js"; + +interface WasmExports { + memory: WebAssembly.Memory; + utf8Count(str: string): number; + utf8Encode(str: string, offset: number): number; + utf8Decode(offset: number, length: number): string; +} + +let wasm: WasmExports | null = null; + +function base64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +// Polyfill for js-string-builtins (used when native builtins unavailable) +const jsStringPolyfill = { + "wasm:js-string": { + length: (s: string) => s.length, + charCodeAt: (s: string, i: number) => s.charCodeAt(i), + codePointAt: (s: string, i: number) => s.codePointAt(i), + fromCharCode: (code: number) => String.fromCharCode(code), + fromCodePoint: (code: number) => String.fromCodePoint(code), + concat: (a: string, b: string) => a + b, + substring: (s: string, start: number, end: number) => s.substring(start, end), + equals: (a: string, b: string) => a === b, + }, +}; + +// Synchronous initialization +function initWasm(): boolean { + if (wasm) return true; + + try { + const bytes = base64ToBytes(wasmBinary); + // Try with builtins first (native support) + // If builtins not supported, option is ignored and polyfill is used + const module = new WebAssembly.Module(bytes, { builtins: ["js-string"] }); + const instance = new WebAssembly.Instance(module, jsStringPolyfill); + wasm = instance.exports as WasmExports; + return true; + } catch { + return false; // Fallback to pure JS (utf8CountJs, etc.) + } +} + +// Try init at module load +const wasmAvailable = initWasm(); + +export function utf8Count(str: string): number { + return wasm ? wasm.utf8Count(str) : utf8CountJs(str); +} +``` + +**Progressive enhancement:** +- Native builtins β†’ engine ignores import object, uses optimized builtins +- No native builtins β†’ engine uses polyfill from import object +- Wasm fails entirely β†’ falls back to pure JS implementation + +**Benefits of base64 inline:** +- No async initialization needed - sync `new WebAssembly.Module()` +- No fetch/network request - works in all environments +- Single file distribution - no separate .wasm asset +- Bundle size: ~1.3x wasm size (base64 overhead), but gzip compresses well + +## Compatibility Matrix + +| Environment | Native builtins | Wasm + polyfill | Pure JS fallback | +|-------------|-----------------|-----------------|------------------| +| Chrome 131+ | Yes | - | - | +| Firefox 134+ | Yes | - | - | +| Safari 18+ | TBD | Yes | - | +| Node.js 24+ | Yes (V8 13.6+) | - | - | +| Node.js 22-23 | Flag required | Yes | - | +| Deno | TBD | Yes | - | +| Older browsers | No | Yes | - | +| No Wasm support | - | - | Yes | + +Three-tier fallback: +1. **Native builtins** - best performance (engine-optimized) +2. **Wasm + polyfill** - good performance (wasm logic, JS string ops) +3. **Pure JS** - baseline (current implementation) + +## Benchmarking Strategy + +1. Reuse existing benchmarks: + - `benchmark/encode-string.ts` + - `benchmark/decode-string.ts` + +2. Add Wasm variants and compare across string sizes: + - Short strings (< 50 bytes): likely JS faster due to call overhead + - Medium strings (50-1000 bytes): Wasm should win + - Large strings (> 1000 bytes): TextEncoder/TextDecoder still optimal + +## Success Criteria + +1. **Performance**: >= 1.5x speedup for medium strings (50-1000 bytes) +2. **Bundle size**: Wasm binary < 2KB (~2.7KB as base64, compresses well with gzip) +3. **Compatibility**: Zero breakage with fallback to pure JS +4. **Maintainability**: Simple WAT, easy to understand + +## Decisions + +- **Node.js**: js-string-builtins enabled by default in Node.js 24+ (V8 13.6+). For Node.js 22-23, use `--experimental-wasm-imported-strings` flag. + +## References + +- [js-string-builtins proposal](https://github.com/WebAssembly/js-string-builtins) +- [MDN: WebAssembly JavaScript builtins](https://developer.mozilla.org/en-US/docs/WebAssembly/Guides/JavaScript_builtins) +- [WebAssembly 3.0 announcement](https://webassembly.org/news/2025-09-17-wasm-3.0/) +- [Previous PR #26](https://github.com/msgpack/msgpack-javascript/pull/26) +- [Removal PR #95](https://github.com/msgpack/msgpack-javascript/pull/95) diff --git a/wasm/build.sh b/wasm/build.sh new file mode 100755 index 00000000..43a4e096 --- /dev/null +++ b/wasm/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Build script for UTF-8 wasm module +# Requires: binaryen (brew install binaryen) + +set -e + +cd "$(dirname "$0")" + +echo "Compiling utf8.wat -> utf8.wasm..." +wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc + +echo "Generating base64 TypeScript module..." +cat > ../src/utils/utf8-wasm-binary.ts << 'HEADER' +// Auto-generated by wasm/build.sh - do not edit manually +// Source: wasm/utf8.wat + +HEADER + +echo -n "export const wasmBinary = \"" >> ../src/utils/utf8-wasm-binary.ts +base64 -i utf8.wasm | tr -d '\n' >> ../src/utils/utf8-wasm-binary.ts +echo "\";" >> ../src/utils/utf8-wasm-binary.ts + +echo "Done! Generated:" +echo " - wasm/utf8.wasm ($(wc -c < utf8.wasm | tr -d ' ') bytes)" +echo " - src/utils/utf8-wasm-binary.ts ($(wc -c < ../src/utils/utf8-wasm-binary.ts | tr -d ' ') bytes)" diff --git a/wasm/utf8.wat b/wasm/utf8.wat new file mode 100644 index 00000000..70349eda --- /dev/null +++ b/wasm/utf8.wat @@ -0,0 +1,244 @@ +;; UTF-8 string processing using js-string-builtins +;; https://github.com/WebAssembly/js-string-builtins + +(module + ;; Import js-string builtins + ;; Note: string parameters use externref, string returns use (ref extern) + (import "wasm:js-string" "length" + (func $str_length (param externref) (result i32))) + (import "wasm:js-string" "charCodeAt" + (func $str_charCodeAt (param externref i32) (result i32))) + (import "wasm:js-string" "fromCharCode" + (func $str_fromCharCode (param i32) (result (ref extern)))) + (import "wasm:js-string" "concat" + (func $str_concat (param externref externref) (result (ref extern)))) + + ;; Linear memory for UTF-8 bytes (64KB initial, exported for JS access) + (memory (export "memory") 1) + + ;; Count UTF-8 byte length of a JS string + ;; This is equivalent to Buffer.byteLength(str, 'utf8') or TextEncoder().encode(str).length + (func (export "utf8Count") (param $str externref) (result i32) + (local $i i32) + (local $len i32) + (local $byteLen i32) + (local $code i32) + + (local.set $len (call $str_length (local.get $str))) + + (block $break + (loop $continue + (br_if $break (i32.ge_u (local.get $i) (local.get $len))) + + (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) + + ;; 1-byte: 0x00-0x7F + (if (i32.lt_u (local.get $code) (i32.const 0x80)) + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 1)))) + ;; 2-byte: 0x80-0x7FF + (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 2)))) + ;; Check for surrogate pair (high surrogate: 0xD800-0xDBFF) + (else (if (i32.and + (i32.ge_u (local.get $code) (i32.const 0xD800)) + (i32.le_u (local.get $code) (i32.const 0xDBFF))) + ;; 4-byte: surrogate pair, skip next char (low surrogate) + (then + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 4))) + (local.set $i (i32.add (local.get $i) (i32.const 1)))) + ;; 3-byte: 0x800-0xFFFF (excluding surrogates) + (else + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 3))))))))) + + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $continue))) + + (local.get $byteLen)) + + ;; Encode JS string to UTF-8 bytes at offset in linear memory + ;; Returns number of bytes written + (func (export "utf8Encode") (param $str externref) (param $offset i32) (result i32) + (local $i i32) + (local $len i32) + (local $pos i32) + (local $code i32) + (local $code2 i32) + + (local.set $len (call $str_length (local.get $str))) + (local.set $pos (local.get $offset)) + + (block $break + (loop $continue + (br_if $break (i32.ge_u (local.get $i) (local.get $len))) + + (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) + + ;; 1-byte: ASCII (0x00-0x7F) + (if (i32.lt_u (local.get $code) (i32.const 0x80)) + (then + (i32.store8 (local.get $pos) (local.get $code)) + (local.set $pos (i32.add (local.get $pos) (i32.const 1)))) + + ;; 2-byte: 0x80-0x7FF + (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) + (then + (i32.store8 (local.get $pos) + (i32.or (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0xC0))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) + (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) + + ;; Check for high surrogate (0xD800-0xDBFF) + (else (if (i32.and + (i32.ge_u (local.get $code) (i32.const 0xD800)) + (i32.le_u (local.get $code) (i32.const 0xDBFF))) + ;; 4-byte: surrogate pair + (then + ;; Get low surrogate + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (local.set $code2 (call $str_charCodeAt (local.get $str) (local.get $i))) + ;; Calculate code point: ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000 + (local.set $code + (i32.add + (i32.add + (i32.shl + (i32.sub (local.get $code) (i32.const 0xD800)) + (i32.const 10)) + (i32.sub (local.get $code2) (i32.const 0xDC00))) + (i32.const 0x10000))) + ;; Encode 4-byte UTF-8 + (i32.store8 (local.get $pos) + (i32.or (i32.shr_u (local.get $code) (i32.const 18)) (i32.const 0xF0))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0x3F)) (i32.const 0x80))) + (i32.store8 (i32.add (local.get $pos) (i32.const 2)) + (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) + (i32.store8 (i32.add (local.get $pos) (i32.const 3)) + (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) + (local.set $pos (i32.add (local.get $pos) (i32.const 4)))) + + ;; 3-byte: 0x800-0xFFFF (excluding surrogates) + (else + (i32.store8 (local.get $pos) + (i32.or (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0xE0))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) + (i32.store8 (i32.add (local.get $pos) (i32.const 2)) + (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) + (local.set $pos (i32.add (local.get $pos) (i32.const 3))))))))) + + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $continue))) + + (i32.sub (local.get $pos) (local.get $offset))) + + ;; Decode UTF-8 bytes from linear memory to JS string + ;; Reads from offset for length bytes + (func (export "utf8Decode") (param $offset i32) (param $length i32) (result externref) + (local $pos i32) + (local $end i32) + (local $result externref) + (local $byte1 i32) + (local $byte2 i32) + (local $byte3 i32) + (local $byte4 i32) + (local $codePoint i32) + + (local.set $pos (local.get $offset)) + (local.set $end (i32.add (local.get $offset) (local.get $length))) + ;; Start with empty string (NUL char, will be trimmed by JS side if needed) + (local.set $result (call $str_fromCharCode (i32.const 0))) + + (block $break + (loop $continue + (br_if $break (i32.ge_u (local.get $pos) (local.get $end))) + + (local.set $byte1 (i32.load8_u (local.get $pos))) + + ;; 1-byte: 0xxxxxxx + (if (i32.eqz (i32.and (local.get $byte1) (i32.const 0x80))) + (then + (local.set $result + (call $str_concat + (local.get $result) + (call $str_fromCharCode (local.get $byte1)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 1))) + (br $continue))) + + ;; 2-byte: 110xxxxx 10xxxxxx + (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xE0)) (i32.const 0xC0)) + (then + (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (local.set $codePoint + (i32.or + (i32.shl (i32.and (local.get $byte1) (i32.const 0x1F)) (i32.const 6)) + (i32.and (local.get $byte2) (i32.const 0x3F)))) + (local.set $result + (call $str_concat + (local.get $result) + (call $str_fromCharCode (local.get $codePoint)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 2))) + (br $continue))) + + ;; 3-byte: 1110xxxx 10xxxxxx 10xxxxxx + (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xF0)) (i32.const 0xE0)) + (then + (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (local.set $byte3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) + (local.set $codePoint + (i32.or + (i32.or + (i32.shl (i32.and (local.get $byte1) (i32.const 0x0F)) (i32.const 12)) + (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 6))) + (i32.and (local.get $byte3) (i32.const 0x3F)))) + (local.set $result + (call $str_concat + (local.get $result) + (call $str_fromCharCode (local.get $codePoint)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 3))) + (br $continue))) + + ;; 4-byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xF8)) (i32.const 0xF0)) + (then + (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (local.set $byte3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) + (local.set $byte4 (i32.load8_u (i32.add (local.get $pos) (i32.const 3)))) + (local.set $codePoint + (i32.or + (i32.or + (i32.or + (i32.shl (i32.and (local.get $byte1) (i32.const 0x07)) (i32.const 18)) + (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 12))) + (i32.shl (i32.and (local.get $byte3) (i32.const 0x3F)) (i32.const 6))) + (i32.and (local.get $byte4) (i32.const 0x3F)))) + ;; Convert to surrogate pair + (local.set $codePoint (i32.sub (local.get $codePoint) (i32.const 0x10000))) + ;; High surrogate + (local.set $result + (call $str_concat + (local.get $result) + (call $str_fromCharCode + (i32.or + (i32.shr_u (local.get $codePoint) (i32.const 10)) + (i32.const 0xD800))))) + ;; Low surrogate + (local.set $result + (call $str_concat + (local.get $result) + (call $str_fromCharCode + (i32.or + (i32.and (local.get $codePoint) (i32.const 0x3FF)) + (i32.const 0xDC00))))) + (local.set $pos (i32.add (local.get $pos) (i32.const 4))) + (br $continue))) + + ;; Invalid byte, skip + (local.set $pos (i32.add (local.get $pos) (i32.const 1))) + (br $continue))) + + ;; Result has leading NUL char - JS side will slice it off + (local.get $result)) +) From 17477102d7b61f6ff5d677df0e80534f78502f0a Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 20:18:30 +0900 Subject: [PATCH 02/13] optimize utf8decode-wasm --- src/utils/utf8-wasm-binary.ts | 2 +- src/utils/utf8-wasm.ts | 25 +++++++++---- test/utf8-wasm.test.ts | 2 +- wasm/utf8.wat | 70 +++++++++++++++++------------------ 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index 4ca1b240..c310ebc3 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -1,4 +1,4 @@ // Auto-generated by wasm/build.sh - do not edit manually // Source: wasm/utf8.wat -export const wasmBinary = "AGFzbQEAAAABHwVgAW8Bf2ACb38Bf2ABfwFkb2ACb28BZG9gAn9/AW8CawQOd2FzbTpqcy1zdHJpbmcGbGVuZ3RoAAAOd2FzbTpqcy1zdHJpbmcKY2hhckNvZGVBdAABDndhc206anMtc3RyaW5nDGZyb21DaGFyQ29kZQACDndhc206anMtc3RyaW5nBmNvbmNhdAADAwQDAAEEBQMBAAEHMAQGbWVtb3J5AgAJdXRmOENvdW50AAQKdXRmOEVuY29kZQAFCnV0ZjhEZWNvZGUABgqXBgN1AQR/IAAQACECAkADQCABIAJPDQEgACABEAEhBCAEQYABSQRAIANBAWohAwUgBEGAEEkEQCADQQJqIQMFIARBgLADTyAEQf+3A01xBEAgA0EEaiEDIAFBAWohAQUgA0EDaiEDCwsLIAFBAWohAQwACwALIAMLvQIBBX8gABAAIQMgASEEAkADQCACIANPDQEgACACEAEhBSAFQYABSQRAIAQgBToAACAEQQFqIQQFIAVBgBBJBEAgBCAFQQZ2QcABcjoAACAEQQFqIAVBP3FBgAFyOgAAIARBAmohBAUgBUGAsANPIAVB/7cDTXEEQCACQQFqIQIgACACEAEhBiAFQYCwA2tBCnQgBkGAuANrakGAgARqIQUgBCAFQRJ2QfABcjoAACAEQQFqIAVBDHZBP3FBgAFyOgAAIARBAmogBUEGdkE/cUGAAXI6AAAgBEEDaiAFQT9xQYABcjoAACAEQQRqIQQFIAQgBUEMdkHgAXI6AAAgBEEBaiAFQQZ2QT9xQYABcjoAACAEQQJqIAVBP3FBgAFyOgAAIARBA2ohBAsLCyACQQFqIQIMAAsACyAEIAFrC98CAgd/AW8gACECIAAgAWohA0EAEAIhCQJAA0AgAiADTw0BIAItAAAhBCAEQYABcUUEQCAJIAQQAhADIQkgAkEBaiECDAELIARB4AFxQcABRgRAIAJBAWotAAAhBSAEQR9xQQZ0IAVBP3FyIQggCSAIEAIQAyEJIAJBAmohAgwBCyAEQfABcUHgAUYEQCACQQFqLQAAIQUgAkECai0AACEGIARBD3FBDHQgBUE/cUEGdHIgBkE/cXIhCCAJIAgQAhADIQkgAkEDaiECDAELIARB+AFxQfABRgRAIAJBAWotAAAhBSACQQJqLQAAIQYgAkEDai0AACEHIARBB3FBEnQgBUE/cUEMdHIgBkE/cUEGdHIgB0E/cXIhCCAIQYCABGshCCAJIAhBCnZBgLADchACEAMhCSAJIAhB/wdxQYC4A3IQAhADIQkgAkEEaiECDAELIAJBAWohAgwACwALIAkL"; +export const wasmBinary = "AGFzbQEAAAABHgVgAW8Bf2ACb38Bf2ABfwFkb2ACb28BZG9gAX8BfwJrBA53YXNtOmpzLXN0cmluZwZsZW5ndGgAAA53YXNtOmpzLXN0cmluZwpjaGFyQ29kZUF0AAEOd2FzbTpqcy1zdHJpbmcMZnJvbUNoYXJDb2RlAAIOd2FzbTpqcy1zdHJpbmcGY29uY2F0AAMDBAMAAQQFAwEAAQYIAX8AQYCAAgsHOAQGbWVtb3J5AgAJdXRmOENvdW50AAQKdXRmOEVuY29kZQAFEnV0ZjhEZWNvZGVUb01lbW9yeQAGCqoGA3UBBH8gABAAIQICQANAIAEgAk8NASAAIAEQASEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAUEBaiEBBSADQQNqIQMLCwsgAUEBaiEBDAALAAsgAwu9AgEFfyAAEAAhAyABIQQCQANAIAIgA08NASAAIAIQASEFIAVBgAFJBEAgBCAFOgAAIARBAWohBAUgBUGAEEkEQCAEIAVBBnZBwAFyOgAAIARBAWogBUE/cUGAAXI6AAAgBEECaiEEBSAFQYCwA08gBUH/twNNcQRAIAJBAWohAiAAIAIQASEGIAVBgLADa0EKdCAGQYC4A2tqQYCABGohBSAEIAVBEnZB8AFyOgAAIARBAWogBUEMdkE/cUGAAXI6AAAgBEECaiAFQQZ2QT9xQYABcjoAACAEQQNqIAVBP3FBgAFyOgAAIARBBGohBAUgBCAFQQx2QeABcjoAACAEQQFqIAVBBnZBP3FBgAFyOgAAIARBAmogBUE/cUGAAXI6AAAgBEEDaiEECwsLIAJBAWohAgwACwALIAQgAWsL8gIBCH9BACEBIAAhAiMAIQMCQANAIAEgAk8NASABLQAAIQQgBEGAAXFFBEAgAyAEOwEAIANBAmohAyABQQFqIQEMAQsgBEHgAXFBwAFGBEAgAUEBai0AACEFIARBH3FBBnQgBUE/cXIhCCADIAg7AQAgA0ECaiEDIAFBAmohAQwBCyAEQfABcUHgAUYEQCABQQFqLQAAIQUgAUECai0AACEGIARBD3FBDHQgBUE/cUEGdHIgBkE/cXIhCCADIAg7AQAgA0ECaiEDIAFBA2ohAQwBCyAEQfgBcUHwAUYEQCABQQFqLQAAIQUgAUECai0AACEGIAFBA2otAAAhByAEQQdxQRJ0IAVBP3FBDHRyIAZBP3FBBnRyIAdBP3FyIQggCEGAgARrIQggAyAIQQp2QYCwA3I7AQAgA0ECaiEDIAMgCEH/B3FBgLgDcjsBACADQQJqIQMgAUEEaiEBDAELIAFBAWohAQwACwALIAMjAGtBAXYL"; diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts index 5fa9976f..fdb2c3d5 100644 --- a/src/utils/utf8-wasm.ts +++ b/src/utils/utf8-wasm.ts @@ -43,9 +43,15 @@ interface WasmExports { memory: WebAssembly.Memory; utf8Count(str: string): number; utf8Encode(str: string, offset: number): number; - utf8Decode(offset: number, length: number): string; + utf8DecodeToMemory(length: number): number; } +// Memory layout constants (must match WAT file) +const UTF16_OFFSET = 32768; // 32KB offset for UTF-16 output + +// Shared TextDecoder for UTF-16LE decoding +const utf16Decoder = new TextDecoder("utf-16le"); + let wasmInstance: WasmExports | null = null; let wasmInitError: Error | null = null; @@ -178,20 +184,25 @@ export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLengt } // Ensure wasm memory is large enough - const requiredPages = Math.ceil(byteLength / 65536); + // Need space for UTF-8 input (0 to byteLength) and UTF-16 output (UTF16_OFFSET onwards) + // Max UTF-16 output is 2 bytes per code unit, and max expansion is 2x (for ASCII) + const utf16MaxBytes = byteLength * 2; + const requiredBytes = UTF16_OFFSET + utf16MaxBytes; + const requiredPages = Math.ceil(requiredBytes / 65536); const currentPages = wasmInstance.memory.buffer.byteLength / 65536; if (requiredPages > currentPages) { wasmInstance.memory.grow(requiredPages - currentPages); } - // Copy bytes to wasm memory + // Copy bytes to wasm memory at offset 0 const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, byteLength); wasmBytes.set(bytes.subarray(inputOffset, inputOffset + byteLength)); - // Decode from wasm memory - const result = wasmInstance.utf8Decode(0, byteLength); + // Decode UTF-8 to UTF-16 in wasm memory, get number of code units + const codeUnits = wasmInstance.utf8DecodeToMemory(byteLength); - // Remove leading NUL character (artifact of wasm implementation) - return result.length > 0 && result.charCodeAt(0) === 0 ? result.slice(1) : result; + // Read UTF-16 code units from wasm memory and decode to string + const utf16Bytes = new Uint8Array(wasmInstance.memory.buffer, UTF16_OFFSET, codeUnits * 2); + return utf16Decoder.decode(utf16Bytes); } diff --git a/test/utf8-wasm.test.ts b/test/utf8-wasm.test.ts index 4ba7355c..47350f37 100644 --- a/test/utf8-wasm.test.ts +++ b/test/utf8-wasm.test.ts @@ -20,7 +20,7 @@ describe("utf8-wasm", () => { assert.ok(exports !== null); assert.ok(typeof exports!.utf8Count === "function"); assert.ok(typeof exports!.utf8Encode === "function"); - assert.ok(typeof exports!.utf8Decode === "function"); + assert.ok(typeof exports!.utf8DecodeToMemory === "function"); assert.ok(exports!.memory instanceof WebAssembly.Memory); } else { assert.strictEqual(exports, null); diff --git a/wasm/utf8.wat b/wasm/utf8.wat index 70349eda..bafd5199 100644 --- a/wasm/utf8.wat +++ b/wasm/utf8.wat @@ -13,9 +13,14 @@ (import "wasm:js-string" "concat" (func $str_concat (param externref externref) (result (ref extern)))) - ;; Linear memory for UTF-8 bytes (64KB initial, exported for JS access) + ;; Linear memory layout: + ;; - 0 to 32KB: UTF-8 input bytes + ;; - 32KB onwards: UTF-16 code units output (i16 array) (memory (export "memory") 1) + ;; Offset where UTF-16 output starts (32KB = 32768) + (global $utf16_offset i32 (i32.const 32768)) + ;; Count UTF-8 byte length of a JS string ;; This is equivalent to Buffer.byteLength(str, 'utf8') or TextEncoder().encode(str).length (func (export "utf8Count") (param $str externref) (result i32) @@ -134,22 +139,23 @@ (i32.sub (local.get $pos) (local.get $offset))) - ;; Decode UTF-8 bytes from linear memory to JS string - ;; Reads from offset for length bytes - (func (export "utf8Decode") (param $offset i32) (param $length i32) (result externref) + ;; Decode UTF-8 bytes to UTF-16 code units in memory + ;; Reads UTF-8 from offset 0 for $length bytes + ;; Writes UTF-16 code units to utf16_offset + ;; Returns number of UTF-16 code units written + (func (export "utf8DecodeToMemory") (param $length i32) (result i32) (local $pos i32) (local $end i32) - (local $result externref) + (local $outPos i32) (local $byte1 i32) (local $byte2 i32) (local $byte3 i32) (local $byte4 i32) (local $codePoint i32) - (local.set $pos (local.get $offset)) - (local.set $end (i32.add (local.get $offset) (local.get $length))) - ;; Start with empty string (NUL char, will be trimmed by JS side if needed) - (local.set $result (call $str_fromCharCode (i32.const 0))) + (local.set $pos (i32.const 0)) + (local.set $end (local.get $length)) + (local.set $outPos (global.get $utf16_offset)) (block $break (loop $continue @@ -160,10 +166,8 @@ ;; 1-byte: 0xxxxxxx (if (i32.eqz (i32.and (local.get $byte1) (i32.const 0x80))) (then - (local.set $result - (call $str_concat - (local.get $result) - (call $str_fromCharCode (local.get $byte1)))) + (i32.store16 (local.get $outPos) (local.get $byte1)) + (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) (local.set $pos (i32.add (local.get $pos) (i32.const 1))) (br $continue))) @@ -175,10 +179,8 @@ (i32.or (i32.shl (i32.and (local.get $byte1) (i32.const 0x1F)) (i32.const 6)) (i32.and (local.get $byte2) (i32.const 0x3F)))) - (local.set $result - (call $str_concat - (local.get $result) - (call $str_fromCharCode (local.get $codePoint)))) + (i32.store16 (local.get $outPos) (local.get $codePoint)) + (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) (local.set $pos (i32.add (local.get $pos) (i32.const 2))) (br $continue))) @@ -193,10 +195,8 @@ (i32.shl (i32.and (local.get $byte1) (i32.const 0x0F)) (i32.const 12)) (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 6))) (i32.and (local.get $byte3) (i32.const 0x3F)))) - (local.set $result - (call $str_concat - (local.get $result) - (call $str_fromCharCode (local.get $codePoint)))) + (i32.store16 (local.get $outPos) (local.get $codePoint)) + (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) (local.set $pos (i32.add (local.get $pos) (i32.const 3))) (br $continue))) @@ -217,21 +217,17 @@ ;; Convert to surrogate pair (local.set $codePoint (i32.sub (local.get $codePoint) (i32.const 0x10000))) ;; High surrogate - (local.set $result - (call $str_concat - (local.get $result) - (call $str_fromCharCode - (i32.or - (i32.shr_u (local.get $codePoint) (i32.const 10)) - (i32.const 0xD800))))) + (i32.store16 (local.get $outPos) + (i32.or + (i32.shr_u (local.get $codePoint) (i32.const 10)) + (i32.const 0xD800))) + (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) ;; Low surrogate - (local.set $result - (call $str_concat - (local.get $result) - (call $str_fromCharCode - (i32.or - (i32.and (local.get $codePoint) (i32.const 0x3FF)) - (i32.const 0xDC00))))) + (i32.store16 (local.get $outPos) + (i32.or + (i32.and (local.get $codePoint) (i32.const 0x3FF)) + (i32.const 0xDC00))) + (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) (local.set $pos (i32.add (local.get $pos) (i32.const 4))) (br $continue))) @@ -239,6 +235,6 @@ (local.set $pos (i32.add (local.get $pos) (i32.const 1))) (br $continue))) - ;; Result has leading NUL char - JS side will slice it off - (local.get $result)) + ;; Return number of UTF-16 code units written + (i32.shr_u (i32.sub (local.get $outPos) (global.get $utf16_offset)) (i32.const 1))) ) From 06034dc7bfb607d4810178c9df2b839909f5ea39 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 22:01:12 +0900 Subject: [PATCH 03/13] optimize wasm impl --- benchmark/decode-string.ts | 7 ++- benchmark/encode-string.ts | 7 ++- src/utils/utf8-wasm-binary.ts | 2 +- src/utils/utf8-wasm.ts | 81 +++++++++++-------------- test/utf8-wasm.test.ts | 4 +- wasm/build.sh | 2 +- wasm/utf8.wat | 109 +++++++++++++++++++++------------- 7 files changed, 119 insertions(+), 93 deletions(-) diff --git a/benchmark/decode-string.ts b/benchmark/decode-string.ts index c02d2041..f9612eab 100644 --- a/benchmark/decode-string.ts +++ b/benchmark/decode-string.ts @@ -5,6 +5,11 @@ import { getWasmError, utf8DecodeWasm } from "../src/utils/utf8-wasm.ts"; // @ts-ignore import Benchmark from "benchmark"; +// description +console.log("utf8DecodeJs - pure JS implementation"); +console.log("utf8DecodeTD - TextDecoder implementation"); +console.log("utf8DecodeWasm - WebAssembly implementation"); + // Show wasm status console.log("=".repeat(60)); console.log("WebAssembly Status:"); @@ -42,7 +47,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { } }); - suite.add("utf8DecodeTD (TextDecoder)", () => { + suite.add("utf8DecodeTD", () => { if (utf8DecodeTD(bytes, 0, byteLength) !== str) { throw new Error("wrong result!"); } diff --git a/benchmark/encode-string.ts b/benchmark/encode-string.ts index ac7ed8dc..3a465290 100644 --- a/benchmark/encode-string.ts +++ b/benchmark/encode-string.ts @@ -5,6 +5,11 @@ import { getWasmError, utf8EncodeWasm } from "../src/utils/utf8-wasm.ts"; // @ts-ignore import Benchmark from "benchmark"; +// description +console.log("utf8EncodeJs - pure JS implementation"); +console.log("utf8EncodeTE - TextEncoder implementation"); +console.log("utf8EncodeWasm - WebAssembly implementation"); + // Show wasm status console.log("=".repeat(60)); console.log("WebAssembly Status:"); @@ -39,7 +44,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { utf8EncodeJs(str, buffer, 0); }); - suite.add("utf8EncodeTE (TextEncoder)", () => { + suite.add("utf8EncodeTE", () => { utf8EncodeTE(str, buffer, 0); }); diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index c310ebc3..b9e641b0 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -1,4 +1,4 @@ // Auto-generated by wasm/build.sh - do not edit manually // Source: wasm/utf8.wat -export const wasmBinary = "AGFzbQEAAAABHgVgAW8Bf2ACb38Bf2ABfwFkb2ACb28BZG9gAX8BfwJrBA53YXNtOmpzLXN0cmluZwZsZW5ndGgAAA53YXNtOmpzLXN0cmluZwpjaGFyQ29kZUF0AAEOd2FzbTpqcy1zdHJpbmcMZnJvbUNoYXJDb2RlAAIOd2FzbTpqcy1zdHJpbmcGY29uY2F0AAMDBAMAAQQFAwEAAQYIAX8AQYCAAgsHOAQGbWVtb3J5AgAJdXRmOENvdW50AAQKdXRmOEVuY29kZQAFEnV0ZjhEZWNvZGVUb01lbW9yeQAGCqoGA3UBBH8gABAAIQICQANAIAEgAk8NASAAIAEQASEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAUEBaiEBBSADQQNqIQMLCwsgAUEBaiEBDAALAAsgAwu9AgEFfyAAEAAhAyABIQQCQANAIAIgA08NASAAIAIQASEFIAVBgAFJBEAgBCAFOgAAIARBAWohBAUgBUGAEEkEQCAEIAVBBnZBwAFyOgAAIARBAWogBUE/cUGAAXI6AAAgBEECaiEEBSAFQYCwA08gBUH/twNNcQRAIAJBAWohAiAAIAIQASEGIAVBgLADa0EKdCAGQYC4A2tqQYCABGohBSAEIAVBEnZB8AFyOgAAIARBAWogBUEMdkE/cUGAAXI6AAAgBEECaiAFQQZ2QT9xQYABcjoAACAEQQNqIAVBP3FBgAFyOgAAIARBBGohBAUgBCAFQQx2QeABcjoAACAEQQFqIAVBBnZBP3FBgAFyOgAAIARBAmogBUE/cUGAAXI6AAAgBEEDaiEECwsLIAJBAWohAgwACwALIAQgAWsL8gIBCH9BACEBIAAhAiMAIQMCQANAIAEgAk8NASABLQAAIQQgBEGAAXFFBEAgAyAEOwEAIANBAmohAyABQQFqIQEMAQsgBEHgAXFBwAFGBEAgAUEBai0AACEFIARBH3FBBnQgBUE/cXIhCCADIAg7AQAgA0ECaiEDIAFBAmohAQwBCyAEQfABcUHgAUYEQCABQQFqLQAAIQUgAUECai0AACEGIARBD3FBDHQgBUE/cUEGdHIgBkE/cXIhCCADIAg7AQAgA0ECaiEDIAFBA2ohAQwBCyAEQfgBcUHwAUYEQCABQQFqLQAAIQUgAUECai0AACEGIAFBA2otAAAhByAEQQdxQRJ0IAVBP3FBDHRyIAZBP3FBBnRyIAdBP3FyIQggCEGAgARrIQggAyAIQQp2QYCwA3I7AQAgA0ECaiEDIAMgCEH/B3FBgLgDcjsBACADQQJqIQMgAUEEaiEBDAELIAFBAWohAQwACwALIAMjAGtBAXYL"; +export const wasmBinary = "AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZAB/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29kZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVAYGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxsb2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwr7BgWUAQIEfwFkACAAEAAhASABRQRAQQAPC0EAIAH7BgAhBSAAIAVBABABGgJAA0AgAiABTw0BIAUgAvsNACEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAkEBaiECBSADQQNqIQMLCwsgAkEBaiECDAALAAsgAwvdAgIFfwFkACAAEAAhAiABIQQgAkUEQEEADwtBACAC+wYAIQcgACAHQQAQARoCQANAIAMgAk8NASAHIAP7DQAhBSAFQYABSQRAIAQgBToAACAEQQFqIQQFIAVBgBBJBEAgBCAFQQZ2QcABcjoAACAEQQFqIAVBP3FBgAFyOgAAIARBAmohBAUgBUGAsANPIAVB/7cDTXEEQCADQQFqIQMgByAD+w0AIQYgBUGAsANrQQp0IAZBgLgDa2pBgIAEaiEFIAQgBUESdkHwAXI6AAAgBEEBaiAFQQx2QT9xQYABcjoAACAEQQJqIAVBBnZBP3FBgAFyOgAAIARBA2ogBUE/cUGAAXI6AAAgBEEEaiEEBSAEIAVBDHZB4AFyOgAAIARBAWogBUEGdkE/cUGAAXI6AAAgBEECaiAFQT9xQYABcjoAACAEQQNqIQQLCwsgA0EBaiEDDAALAAsgBCABawvuAgEIfyAAIQMCQANAIAIgA08NASACLQAAIQUgBUGAAXFFBEAgASAEIAX7DgAgBEEBaiEEIAJBAWohAgwBCyAFQeABcUHAAUYEQCACQQFqLQAAIQYgBUEfcUEGdCAGQT9xciEJIAEgBCAJ+w4AIARBAWohBCACQQJqIQIMAQsgBUHwAXFB4AFGBEAgAkEBai0AACEGIAJBAmotAAAhByAFQQ9xQQx0IAZBP3FBBnRyIAdBP3FyIQkgASAEIAn7DgAgBEEBaiEEIAJBA2ohAgwBCyAFQfgBcUHwAUYEQCACQQFqLQAAIQYgAkECai0AACEHIAJBA2otAAAhCCAFQQdxQRJ0IAZBP3FBDHRyIAdBP3FBBnRyIAhBP3FyIQkgCUGAgARrIQkgASAEIAlBCnZBgLADcvsOACAEQQFqIQQgASAEIAlB/wdxQYC4A3L7DgAgBEEBaiEEIAJBBGohAgwBCyACQQFqIQIMAAsACyAECwkAQQAgAPsGAAsKACAAIAEgAhACCw=="; diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts index fdb2c3d5..3abb40d8 100644 --- a/src/utils/utf8-wasm.ts +++ b/src/utils/utf8-wasm.ts @@ -1,14 +1,12 @@ /** - * WebAssembly-based UTF-8 string processing using js-string-builtins. + * WebAssembly-based UTF-8 string processing using js-string-builtins with GC arrays. * * Environment variables: * - MSGPACK_WASM=force: Force wasm mode, throw error if wasm fails to load * - MSGPACK_WASM=never: Disable wasm, always use pure JS * - * Three-tier fallback: - * 1. Native js-string-builtins (Chrome 130+, Firefox 134+) - * 2. Wasm + polyfill (older browsers with WebAssembly) - * 3. Pure JS (no WebAssembly support) + * This implementation uses WASM GC arrays with intoCharCodeArray/fromCharCodeArray + * for efficient bulk string operations instead of character-by-character processing. */ import { wasmBinary } from "./utf8-wasm-binary.ts"; @@ -39,19 +37,18 @@ function getWasmMode(): "force" | "never" | "auto" { const WASM_MODE = getWasmMode(); +// GC array type (opaque reference) +type I16Array = object; + interface WasmExports { memory: WebAssembly.Memory; utf8Count(str: string): number; utf8Encode(str: string, offset: number): number; - utf8DecodeToMemory(length: number): number; + utf8DecodeToArray(length: number, arr: I16Array): number; + allocArray(size: number): I16Array; + arrayToString(arr: I16Array, start: number, end: number): string; } -// Memory layout constants (must match WAT file) -const UTF16_OFFSET = 32768; // 32KB offset for UTF-16 output - -// Shared TextDecoder for UTF-16LE decoding -const utf16Decoder = new TextDecoder("utf-16le"); - let wasmInstance: WasmExports | null = null; let wasmInitError: Error | null = null; @@ -68,17 +65,6 @@ function base64ToBytes(base64: string): Uint8Array { return new Uint8Array(Buffer.from(base64, "base64")); } -// Polyfill for js-string-builtins (used when native builtins unavailable) -const jsStringPolyfill = { - // eslint-disable-next-line @typescript-eslint/naming-convention - "wasm:js-string": { - length: (s: string) => s.length, - charCodeAt: (s: string, i: number) => s.charCodeAt(i), - fromCharCode: (code: number) => String.fromCharCode(code), - concat: (a: string, b: string) => a + b, - }, -}; - function tryInitWasm(): void { if (wasmInstance !== null || wasmInitError !== null) { return; // Already initialized or failed @@ -96,14 +82,9 @@ function tryInitWasm(): void { const bytes = base64ToBytes(wasmBinary); - // Try with builtins option (native support) - // If builtins not supported, option is ignored and polyfill is used - - + // Requires js-string builtins support (Node.js 24+ / Chrome 130+ / Firefox 134+) const module: WebAssembly.Module = new (WebAssembly.Module as any)(bytes, { builtins: ["js-string"] }); - - - const instance = new (WebAssembly.Instance)(module, jsStringPolyfill); + const instance = new WebAssembly.Instance(module); wasmInstance = instance.exports as unknown as WasmExports; } catch (e) { wasmInitError = e instanceof Error ? e : new Error(String(e)); @@ -121,7 +102,7 @@ tryInitWasm(); * Whether wasm is available and initialized. */ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -export const WASM_AVAILABLE = (wasmInstance !== null); +export const WASM_AVAILABLE = wasmInstance !== null; /** * Get the wasm initialization error, if any. @@ -156,16 +137,20 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu throw new Error("wasm not initialized"); } + // Estimate max byte length without a full pass over the string. + // Each UTF-16 code unit can produce at most 3 UTF-8 bytes (BMP chars). + // Surrogate pairs (2 code units) produce 4 bytes, so 3 bytes/code unit is safe. + const maxByteLength = str.length * 3; + // Ensure wasm memory is large enough - const byteLength = wasmInstance.utf8Count(str); - const requiredPages = Math.ceil((outputOffset + byteLength) / 65536); + const requiredPages = Math.ceil(maxByteLength / 65536); const currentPages = wasmInstance.memory.buffer.byteLength / 65536; if (requiredPages > currentPages) { wasmInstance.memory.grow(requiredPages - currentPages); } - // Encode to wasm memory + // Encode to wasm memory (uses intoCharCodeArray for bulk char extraction) const bytesWritten = wasmInstance.utf8Encode(str, 0); // Copy from wasm memory to output buffer @@ -177,32 +162,36 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu /** * Decode UTF-8 bytes to string. + * Uses GC arrays with fromCharCodeArray for efficient string creation. */ export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { if (wasmInstance === null) { throw new Error("wasm not initialized"); } - // Ensure wasm memory is large enough - // Need space for UTF-8 input (0 to byteLength) and UTF-16 output (UTF16_OFFSET onwards) - // Max UTF-16 output is 2 bytes per code unit, and max expansion is 2x (for ASCII) - const utf16MaxBytes = byteLength * 2; - const requiredBytes = UTF16_OFFSET + utf16MaxBytes; - const requiredPages = Math.ceil(requiredBytes / 65536); + // Handle empty input + if (byteLength === 0) { + return ""; + } + + // Ensure wasm memory is large enough for UTF-8 input + const requiredPages = Math.ceil(byteLength / 65536); const currentPages = wasmInstance.memory.buffer.byteLength / 65536; if (requiredPages > currentPages) { wasmInstance.memory.grow(requiredPages - currentPages); } - // Copy bytes to wasm memory at offset 0 + // Copy UTF-8 bytes to wasm linear memory at offset 0 const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, byteLength); wasmBytes.set(bytes.subarray(inputOffset, inputOffset + byteLength)); - // Decode UTF-8 to UTF-16 in wasm memory, get number of code units - const codeUnits = wasmInstance.utf8DecodeToMemory(byteLength); + // Allocate GC array for UTF-16 output (max size = byteLength for ASCII) + const arr = wasmInstance.allocArray(byteLength); + + // Decode UTF-8 to UTF-16 in GC array + const codeUnits = wasmInstance.utf8DecodeToArray(byteLength, arr); - // Read UTF-16 code units from wasm memory and decode to string - const utf16Bytes = new Uint8Array(wasmInstance.memory.buffer, UTF16_OFFSET, codeUnits * 2); - return utf16Decoder.decode(utf16Bytes); + // Create string directly from GC array using fromCharCodeArray + return wasmInstance.arrayToString(arr, 0, codeUnits); } diff --git a/test/utf8-wasm.test.ts b/test/utf8-wasm.test.ts index 47350f37..58c89687 100644 --- a/test/utf8-wasm.test.ts +++ b/test/utf8-wasm.test.ts @@ -20,7 +20,9 @@ describe("utf8-wasm", () => { assert.ok(exports !== null); assert.ok(typeof exports!.utf8Count === "function"); assert.ok(typeof exports!.utf8Encode === "function"); - assert.ok(typeof exports!.utf8DecodeToMemory === "function"); + assert.ok(typeof exports!.utf8DecodeToArray === "function"); + assert.ok(typeof exports!.allocArray === "function"); + assert.ok(typeof exports!.arrayToString === "function"); assert.ok(exports!.memory instanceof WebAssembly.Memory); } else { assert.strictEqual(exports, null); diff --git a/wasm/build.sh b/wasm/build.sh index 43a4e096..a8211af7 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -7,7 +7,7 @@ set -e cd "$(dirname "$0")" echo "Compiling utf8.wat -> utf8.wasm..." -wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc +wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc --enable-strings echo "Generating base64 TypeScript module..." cat > ../src/utils/utf8-wasm-binary.ts << 'HEADER' diff --git a/wasm/utf8.wat b/wasm/utf8.wat index bafd5199..75c7aaf3 100644 --- a/wasm/utf8.wat +++ b/wasm/utf8.wat @@ -1,41 +1,49 @@ -;; UTF-8 string processing using js-string-builtins +;; UTF-8 string processing using js-string-builtins with GC arrays ;; https://github.com/WebAssembly/js-string-builtins +;; +;; This implementation uses WASM GC arrays with intoCharCodeArray/fromCharCodeArray +;; for efficient bulk string operations instead of character-by-character processing. (module + ;; Define i16 array type for UTF-16 code units + (type $i16_array (array (mut i16))) + ;; Import js-string builtins - ;; Note: string parameters use externref, string returns use (ref extern) (import "wasm:js-string" "length" (func $str_length (param externref) (result i32))) - (import "wasm:js-string" "charCodeAt" - (func $str_charCodeAt (param externref i32) (result i32))) - (import "wasm:js-string" "fromCharCode" - (func $str_fromCharCode (param i32) (result (ref extern)))) - (import "wasm:js-string" "concat" - (func $str_concat (param externref externref) (result (ref extern)))) - - ;; Linear memory layout: - ;; - 0 to 32KB: UTF-8 input bytes - ;; - 32KB onwards: UTF-16 code units output (i16 array) - (memory (export "memory") 1) + (import "wasm:js-string" "intoCharCodeArray" + (func $str_into_array (param externref (ref $i16_array) i32) (result i32))) + (import "wasm:js-string" "fromCharCodeArray" + (func $str_from_array (param (ref $i16_array) i32 i32) (result (ref extern)))) - ;; Offset where UTF-16 output starts (32KB = 32768) - (global $utf16_offset i32 (i32.const 32768)) + ;; Linear memory for UTF-8 bytes (64KB initial, exported for JS access) + (memory (export "memory") 1) ;; Count UTF-8 byte length of a JS string - ;; This is equivalent to Buffer.byteLength(str, 'utf8') or TextEncoder().encode(str).length + ;; Uses GC array to get all char codes at once (func (export "utf8Count") (param $str externref) (result i32) - (local $i i32) (local $len i32) + (local $arr (ref $i16_array)) + (local $i i32) (local $byteLen i32) (local $code i32) (local.set $len (call $str_length (local.get $str))) + ;; Handle empty string + (if (i32.eqz (local.get $len)) + (then (return (i32.const 0)))) + + ;; Allocate array and copy string chars + (local.set $arr (array.new $i16_array (i32.const 0) (local.get $len))) + (drop (call $str_into_array (local.get $str) (local.get $arr) (i32.const 0))) + + ;; Count UTF-8 bytes (block $break (loop $continue (br_if $break (i32.ge_u (local.get $i) (local.get $len))) - (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) + (local.set $code (array.get_u $i16_array (local.get $arr) (local.get $i))) ;; 1-byte: 0x00-0x7F (if (i32.lt_u (local.get $code) (i32.const 0x80)) @@ -64,9 +72,11 @@ ;; Encode JS string to UTF-8 bytes at offset in linear memory ;; Returns number of bytes written + ;; Uses intoCharCodeArray for bulk char code extraction (func (export "utf8Encode") (param $str externref) (param $offset i32) (result i32) - (local $i i32) (local $len i32) + (local $arr (ref $i16_array)) + (local $i i32) (local $pos i32) (local $code i32) (local $code2 i32) @@ -74,11 +84,20 @@ (local.set $len (call $str_length (local.get $str))) (local.set $pos (local.get $offset)) + ;; Handle empty string + (if (i32.eqz (local.get $len)) + (then (return (i32.const 0)))) + + ;; Allocate array and copy all char codes at once + (local.set $arr (array.new $i16_array (i32.const 0) (local.get $len))) + (drop (call $str_into_array (local.get $str) (local.get $arr) (i32.const 0))) + + ;; Encode to UTF-8 (block $break (loop $continue (br_if $break (i32.ge_u (local.get $i) (local.get $len))) - (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) + (local.set $code (array.get_u $i16_array (local.get $arr) (local.get $i))) ;; 1-byte: ASCII (0x00-0x7F) (if (i32.lt_u (local.get $code) (i32.const 0x80)) @@ -101,9 +120,9 @@ (i32.le_u (local.get $code) (i32.const 0xDBFF))) ;; 4-byte: surrogate pair (then - ;; Get low surrogate + ;; Get low surrogate from array (local.set $i (i32.add (local.get $i) (i32.const 1))) - (local.set $code2 (call $str_charCodeAt (local.get $str) (local.get $i))) + (local.set $code2 (array.get_u $i16_array (local.get $arr) (local.get $i))) ;; Calculate code point: ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000 (local.set $code (i32.add @@ -139,23 +158,21 @@ (i32.sub (local.get $pos) (local.get $offset))) - ;; Decode UTF-8 bytes to UTF-16 code units in memory - ;; Reads UTF-8 from offset 0 for $length bytes - ;; Writes UTF-16 code units to utf16_offset - ;; Returns number of UTF-16 code units written - (func (export "utf8DecodeToMemory") (param $length i32) (result i32) + ;; Decode UTF-8 bytes from linear memory to JS string + ;; Uses fromCharCodeArray for direct string creation + ;; Returns: (codeUnitsWritten << 16) | 0 for success, packed in i32 + ;; The actual string is returned via a separate export + (func (export "utf8DecodeToArray") (param $length i32) (param $arr (ref $i16_array)) (result i32) (local $pos i32) (local $end i32) - (local $outPos i32) + (local $outIdx i32) (local $byte1 i32) (local $byte2 i32) (local $byte3 i32) (local $byte4 i32) (local $codePoint i32) - (local.set $pos (i32.const 0)) (local.set $end (local.get $length)) - (local.set $outPos (global.get $utf16_offset)) (block $break (loop $continue @@ -166,8 +183,8 @@ ;; 1-byte: 0xxxxxxx (if (i32.eqz (i32.and (local.get $byte1) (i32.const 0x80))) (then - (i32.store16 (local.get $outPos) (local.get $byte1)) - (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $byte1)) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) (local.set $pos (i32.add (local.get $pos) (i32.const 1))) (br $continue))) @@ -179,8 +196,8 @@ (i32.or (i32.shl (i32.and (local.get $byte1) (i32.const 0x1F)) (i32.const 6)) (i32.and (local.get $byte2) (i32.const 0x3F)))) - (i32.store16 (local.get $outPos) (local.get $codePoint)) - (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $codePoint)) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) (local.set $pos (i32.add (local.get $pos) (i32.const 2))) (br $continue))) @@ -195,8 +212,8 @@ (i32.shl (i32.and (local.get $byte1) (i32.const 0x0F)) (i32.const 12)) (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 6))) (i32.and (local.get $byte3) (i32.const 0x3F)))) - (i32.store16 (local.get $outPos) (local.get $codePoint)) - (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $codePoint)) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) (local.set $pos (i32.add (local.get $pos) (i32.const 3))) (br $continue))) @@ -217,17 +234,17 @@ ;; Convert to surrogate pair (local.set $codePoint (i32.sub (local.get $codePoint) (i32.const 0x10000))) ;; High surrogate - (i32.store16 (local.get $outPos) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (i32.or (i32.shr_u (local.get $codePoint) (i32.const 10)) (i32.const 0xD800))) - (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) ;; Low surrogate - (i32.store16 (local.get $outPos) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (i32.or (i32.and (local.get $codePoint) (i32.const 0x3FF)) (i32.const 0xDC00))) - (local.set $outPos (i32.add (local.get $outPos) (i32.const 2))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) (local.set $pos (i32.add (local.get $pos) (i32.const 4))) (br $continue))) @@ -235,6 +252,14 @@ (local.set $pos (i32.add (local.get $pos) (i32.const 1))) (br $continue))) - ;; Return number of UTF-16 code units written - (i32.shr_u (i32.sub (local.get $outPos) (global.get $utf16_offset)) (i32.const 1))) + ;; Return number of code units written + (local.get $outIdx)) + + ;; Allocate a GC array for UTF-16 code units + (func (export "allocArray") (param $size i32) (result (ref $i16_array)) + (array.new $i16_array (i32.const 0) (local.get $size))) + + ;; Create string from GC array + (func (export "arrayToString") (param $arr (ref $i16_array)) (param $start i32) (param $end i32) (result externref) + (call $str_from_array (local.get $arr) (local.get $start) (local.get $end))) ) From f709bfb00ebf14037f0085cd5c6d7b8a8d73a866 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 22:15:14 +0900 Subject: [PATCH 04/13] optimize wasm build --- src/utils/utf8-wasm-binary.ts | 2 +- wasm/build.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index b9e641b0..2e2f5413 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -1,4 +1,4 @@ // Auto-generated by wasm/build.sh - do not edit manually // Source: wasm/utf8.wat -export const wasmBinary = "AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZAB/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29kZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVAYGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxsb2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwr7BgWUAQIEfwFkACAAEAAhASABRQRAQQAPC0EAIAH7BgAhBSAAIAVBABABGgJAA0AgAiABTw0BIAUgAvsNACEEIARBgAFJBEAgA0EBaiEDBSAEQYAQSQRAIANBAmohAwUgBEGAsANPIARB/7cDTXEEQCADQQRqIQMgAkEBaiECBSADQQNqIQMLCwsgAkEBaiECDAALAAsgAwvdAgIFfwFkACAAEAAhAiABIQQgAkUEQEEADwtBACAC+wYAIQcgACAHQQAQARoCQANAIAMgAk8NASAHIAP7DQAhBSAFQYABSQRAIAQgBToAACAEQQFqIQQFIAVBgBBJBEAgBCAFQQZ2QcABcjoAACAEQQFqIAVBP3FBgAFyOgAAIARBAmohBAUgBUGAsANPIAVB/7cDTXEEQCADQQFqIQMgByAD+w0AIQYgBUGAsANrQQp0IAZBgLgDa2pBgIAEaiEFIAQgBUESdkHwAXI6AAAgBEEBaiAFQQx2QT9xQYABcjoAACAEQQJqIAVBBnZBP3FBgAFyOgAAIARBA2ogBUE/cUGAAXI6AAAgBEEEaiEEBSAEIAVBDHZB4AFyOgAAIARBAWogBUEGdkE/cUGAAXI6AAAgBEECaiAFQT9xQYABcjoAACAEQQNqIQQLCwsgA0EBaiEDDAALAAsgBCABawvuAgEIfyAAIQMCQANAIAIgA08NASACLQAAIQUgBUGAAXFFBEAgASAEIAX7DgAgBEEBaiEEIAJBAWohAgwBCyAFQeABcUHAAUYEQCACQQFqLQAAIQYgBUEfcUEGdCAGQT9xciEJIAEgBCAJ+w4AIARBAWohBCACQQJqIQIMAQsgBUHwAXFB4AFGBEAgAkEBai0AACEGIAJBAmotAAAhByAFQQ9xQQx0IAZBP3FBBnRyIAdBP3FyIQkgASAEIAn7DgAgBEEBaiEEIAJBA2ohAgwBCyAFQfgBcUHwAUYEQCACQQFqLQAAIQYgAkECai0AACEHIAJBA2otAAAhCCAFQQdxQRJ0IAZBP3FBDHRyIAdBP3FBBnRyIAhBP3FyIQkgCUGAgARrIQkgASAEIAlBCnZBgLADcvsOACAEQQFqIQQgASAEIAlB/wdxQYC4A3L7DgAgBEEBaiEEIAJBBGohAgwBCyACQQFqIQIMAAsACyAECwkAQQAgAPsGAAsKACAAIAEgAhACCw=="; +export const wasmBinary = "AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZAB/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29kZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVAYGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxsb2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqZBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIgVBABABGgNAIAEgBE9FBEAgBSAB+w0AIgNBgAFJBH8gAkEBagUgA0GAEEkEfyACQQJqBSADQf+3A00gA0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7wCAgR/AWQAIAEhAiAAEAAiBUUEQEEADwsgACAF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAAIAJBAWoFIANBgBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTSADQYCwA09xBH8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZBP3FBgAFyOgAAIAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDHZB4AFyOgAAIAJBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEBaiEEDAELCyACIAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIAJBAWohAgwCCyAEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkECaiECDAILIARB8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w4AIANBAWohAyACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFqLQAAQT9xQQx0ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3FBgLgDcvsOACADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQAgs="; diff --git a/wasm/build.sh b/wasm/build.sh index a8211af7..44e4f4df 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -8,6 +8,8 @@ cd "$(dirname "$0")" echo "Compiling utf8.wat -> utf8.wasm..." wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc --enable-strings +wasm-opt --enable-gc --enable-strings --enable-reference-types -O4 utf8.wasm -o tmp.wasm +mv tmp.wasm utf8.wasm echo "Generating base64 TypeScript module..." cat > ../src/utils/utf8-wasm-binary.ts << 'HEADER' From 77da497be30950c6be2e6b283969e6c4af470b44 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 22:28:03 +0900 Subject: [PATCH 05/13] tweaks --- src/utils/utf8-wasm-binary.ts | 24 ++++++++++++++-- src/utils/utf8-wasm.ts | 54 +++++++++++------------------------ wasm/build.sh | 16 +++++------ 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index 2e2f5413..2e80eb25 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -1,4 +1,24 @@ -// Auto-generated by wasm/build.sh - do not edit manually +// Auto-generated by wasm/build.sh - DO NOT EDIT MANUALLY // Source: wasm/utf8.wat -export const wasmBinary = "AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZAB/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29kZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVAYGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxsb2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqZBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIgVBABABGgNAIAEgBE9FBEAgBSAB+w0AIgNBgAFJBH8gAkEBagUgA0GAEEkEfyACQQJqBSADQf+3A00gA0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7wCAgR/AWQAIAEhAiAAEAAiBUUEQEEADwsgACAF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAAIAJBAWoFIANBgBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTSADQYCwA09xBH8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZBP3FBgAFyOgAAIAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDHZB4AFyOgAAIAJBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEBaiEEDAELCyACIAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIAJBAWohAgwCCyAEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkECaiECDAILIARB8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w4AIANBAWohAyACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFqLQAAQT9xQQx0ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3FBgLgDcvsOACADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQAgs="; +export const wasmBinary = ` +AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZA +B/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29k +ZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVA +YGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxs +b2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqZBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIg +VBABABGgNAIAEgBE9FBEAgBSAB+w0AIgNBgAFJBH8gAkEBagUgA0GAEEkEfyACQQJqBSADQf+3A00g +A0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7wCAgR/AWQAIAEhAi +AAEAAiBUUEQEEADwsgACAF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAA +IAJBAWoFIANBgBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTS +ADQYCwA09xBH8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZB +P3FBgAFyOgAAIAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDH +ZB4AFyOgAAIAJBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEB +aiEEDAELCyACIAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIA +JBAWohAgwCCyAEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkEC +aiECDAILIARB8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w +4AIANBAWohAyACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFq +LQAAQT9xQQx0ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3 +FBgLgDcvsOACADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQ +Ags= +`; diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts index 3abb40d8..626e2578 100644 --- a/src/utils/utf8-wasm.ts +++ b/src/utils/utf8-wasm.ts @@ -40,7 +40,7 @@ const WASM_MODE = getWasmMode(); // GC array type (opaque reference) type I16Array = object; -interface WasmExports { +interface WasmExports extends WebAssembly.Exports { memory: WebAssembly.Memory; utf8Count(str: string): number; utf8Encode(str: string, offset: number): number; @@ -65,11 +65,7 @@ function base64ToBytes(base64: string): Uint8Array { return new Uint8Array(Buffer.from(base64, "base64")); } -function tryInitWasm(): void { - if (wasmInstance !== null || wasmInitError !== null) { - return; // Already initialized or failed - } - +function tryInitializeWasmInstance(): void { if (WASM_MODE === "never") { wasmInitError = new Error("MSGPACK_WASM=never: wasm disabled"); return; @@ -85,18 +81,17 @@ function tryInitWasm(): void { // Requires js-string builtins support (Node.js 24+ / Chrome 130+ / Firefox 134+) const module: WebAssembly.Module = new (WebAssembly.Module as any)(bytes, { builtins: ["js-string"] }); const instance = new WebAssembly.Instance(module); - wasmInstance = instance.exports as unknown as WasmExports; + wasmInstance = instance.exports as WasmExports; } catch (e) { wasmInitError = e instanceof Error ? e : new Error(String(e)); if (WASM_MODE === "force") { - throw new Error(`MSGPACK_WASM=force but wasm failed to load: ${wasmInitError.message}`); + throw new Error(`MSGPACK_WASM=force but wasm failed to load: ${wasmInitError.message}`, { cause: wasmInitError }); } } } -// Initialize on module load -tryInitWasm(); +tryInitializeWasmInstance(); /** * Whether wasm is available and initialized. @@ -104,16 +99,10 @@ tryInitWasm(); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition export const WASM_AVAILABLE = wasmInstance !== null; -/** - * Get the wasm initialization error, if any. - */ export function getWasmError(): Error | null { return wasmInitError; } -/** - * Get the raw wasm exports for advanced usage. - */ export function getWasmExports(): WasmExports | null { return wasmInstance; } @@ -122,10 +111,7 @@ export function getWasmExports(): WasmExports | null { * Count UTF-8 byte length of a string. */ export function utf8CountWasm(str: string): number { - if (wasmInstance === null) { - throw new Error("wasm not initialized"); - } - return wasmInstance.utf8Count(str); + return wasmInstance!.utf8Count(str); } /** @@ -133,10 +119,6 @@ export function utf8CountWasm(str: string): number { * Returns the number of bytes written. */ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: number): number { - if (wasmInstance === null) { - throw new Error("wasm not initialized"); - } - // Estimate max byte length without a full pass over the string. // Each UTF-16 code unit can produce at most 3 UTF-8 bytes (BMP chars). // Surrogate pairs (2 code units) produce 4 bytes, so 3 bytes/code unit is safe. @@ -144,17 +126,17 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu // Ensure wasm memory is large enough const requiredPages = Math.ceil(maxByteLength / 65536); - const currentPages = wasmInstance.memory.buffer.byteLength / 65536; + const currentPages = wasmInstance!.memory.buffer.byteLength / 65536; if (requiredPages > currentPages) { - wasmInstance.memory.grow(requiredPages - currentPages); + wasmInstance!.memory.grow(requiredPages - currentPages); } // Encode to wasm memory (uses intoCharCodeArray for bulk char extraction) - const bytesWritten = wasmInstance.utf8Encode(str, 0); + const bytesWritten = wasmInstance!.utf8Encode(str, 0); // Copy from wasm memory to output buffer - const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, bytesWritten); + const wasmBytes = new Uint8Array(wasmInstance!.memory.buffer, 0, bytesWritten); output.set(wasmBytes, outputOffset); return bytesWritten; @@ -165,10 +147,6 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu * Uses GC arrays with fromCharCodeArray for efficient string creation. */ export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { - if (wasmInstance === null) { - throw new Error("wasm not initialized"); - } - // Handle empty input if (byteLength === 0) { return ""; @@ -176,22 +154,22 @@ export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLengt // Ensure wasm memory is large enough for UTF-8 input const requiredPages = Math.ceil(byteLength / 65536); - const currentPages = wasmInstance.memory.buffer.byteLength / 65536; + const currentPages = wasmInstance!.memory.buffer.byteLength / 65536; if (requiredPages > currentPages) { - wasmInstance.memory.grow(requiredPages - currentPages); + wasmInstance!.memory.grow(requiredPages - currentPages); } // Copy UTF-8 bytes to wasm linear memory at offset 0 - const wasmBytes = new Uint8Array(wasmInstance.memory.buffer, 0, byteLength); + const wasmBytes = new Uint8Array(wasmInstance!.memory.buffer, 0, byteLength); wasmBytes.set(bytes.subarray(inputOffset, inputOffset + byteLength)); // Allocate GC array for UTF-16 output (max size = byteLength for ASCII) - const arr = wasmInstance.allocArray(byteLength); + const arr = wasmInstance!.allocArray(byteLength); // Decode UTF-8 to UTF-16 in GC array - const codeUnits = wasmInstance.utf8DecodeToArray(byteLength, arr); + const codeUnits = wasmInstance!.utf8DecodeToArray(byteLength, arr); // Create string directly from GC array using fromCharCodeArray - return wasmInstance.arrayToString(arr, 0, codeUnits); + return wasmInstance!.arrayToString(arr, 0, codeUnits); } diff --git a/wasm/build.sh b/wasm/build.sh index 44e4f4df..819dd1db 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -12,15 +12,15 @@ wasm-opt --enable-gc --enable-strings --enable-reference-types -O4 utf8.wasm -o mv tmp.wasm utf8.wasm echo "Generating base64 TypeScript module..." -cat > ../src/utils/utf8-wasm-binary.ts << 'HEADER' -// Auto-generated by wasm/build.sh - do not edit manually -// Source: wasm/utf8.wat -HEADER - -echo -n "export const wasmBinary = \"" >> ../src/utils/utf8-wasm-binary.ts -base64 -i utf8.wasm | tr -d '\n' >> ../src/utils/utf8-wasm-binary.ts -echo "\";" >> ../src/utils/utf8-wasm-binary.ts +{ + echo "// Auto-generated by wasm/build.sh - DO NOT EDIT MANUALLY" + echo "// Source: wasm/utf8.wat" + echo "" + echo "export const wasmBinary = \`" + base64 -b 78 -i utf8.wasm + echo "\`;" +} > ../src/utils/utf8-wasm-binary.ts echo "Done! Generated:" echo " - wasm/utf8.wasm ($(wc -c < utf8.wasm | tr -d ' ') bytes)" From 9fff56736eb7999f991da52ebaf80ac7bddea8d2 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 22:40:44 +0900 Subject: [PATCH 06/13] tweak of thresholds, based on benchmarks --- src/utils/utf8.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts index bb911e3a..7fbd5394 100644 --- a/src/utils/utf8.ts +++ b/src/utils/utf8.ts @@ -98,15 +98,18 @@ const sharedTextEncoder = new TextEncoder(); // This threshold should be determined by benchmarking, which might vary in engines and input data. // Run `npx ts-node benchmark/encode-string.ts` for details. +// For mixed content (ASCII + CJK + emoji), JS wins for strLength < 30-50. +// After that, WASM or TextEncoder is faster depending on content type. const TEXT_ENCODER_THRESHOLD = 50; export function utf8EncodeTE(str: string, output: Uint8Array, outputOffset: number): void { sharedTextEncoder.encodeInto(str, output.subarray(outputOffset)); } -// Wasm threshold: use wasm for medium strings, TextEncoder for large strings -// These thresholds should be determined by benchmarking. -// Run `npx ts-node benchmark/encode-string.ts` for details. +// Wasm threshold: use wasm for medium strings, TextEncoder for large strings. +// For pure ASCII, TextEncoder is ~1.7x faster at 100+ strLength. +// For CJK/emoji, WASM is ~1.4-1.6x faster than TextEncoder at all sizes. +// 1000 is a compromise for mixed content. const WASM_ENCODE_MAX = 1000; function utf8EncodeWithWasm(str: string, output: Uint8Array, outputOffset: number): void { @@ -187,14 +190,19 @@ const sharedTextDecoder = new TextDecoder(); // This threshold should be determined by benchmarking, which might vary in engines and input data. // Run `npx ts-node benchmark/decode-string.ts` for details. -const TEXT_DECODER_THRESHOLD = 200; +// For mixed content (ASCII + CJK + emoji), JS wins for very short strings only. +// WASM becomes superior at ~30-50 bytes for non-ASCII content. +const TEXT_DECODER_THRESHOLD = 50; export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: number): string { const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength); return sharedTextDecoder.decode(stringBytes); } -// Wasm decode threshold: use wasm for medium strings, TextDecoder for large strings +// Wasm decode threshold: use wasm for medium strings, TextDecoder for large strings. +// For pure ASCII, TextDecoder is ~5x faster at 1000+ bytes. +// For CJK/emoji, WASM is ~5-6x faster than TextDecoder at all sizes. +// 1000 is a compromise for mixed content. const WASM_DECODE_MAX = 1000; function utf8DecodeWithWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { From feba4e2f37d9f94014c7789a54d912d23320cc39 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 22:54:14 +0900 Subject: [PATCH 07/13] remove redundant 0 checks --- src/utils/utf8-wasm-binary.ts | 27 +++++++++++++-------------- src/utils/utf8-wasm.ts | 5 ----- wasm/utf8.wat | 4 ---- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index 2e80eb25..8e46f56b 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -6,19 +6,18 @@ AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZA B/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29k ZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVA YGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxs -b2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqZBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIg +b2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqQBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIg VBABABGgNAIAEgBE9FBEAgBSAB+w0AIgNBgAFJBH8gAkEBagUgA0GAEEkEfyACQQJqBSADQf+3A00g -A0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7wCAgR/AWQAIAEhAi -AAEAAiBUUEQEEADwsgACAF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAA -IAJBAWoFIANBgBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTS -ADQYCwA09xBH8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZB -P3FBgAFyOgAAIAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDH -ZB4AFyOgAAIAJBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEB -aiEEDAELCyACIAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIA -JBAWohAgwCCyAEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkEC -aiECDAILIARB8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w -4AIANBAWohAyACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFq -LQAAQT9xQQx0ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3 -FBgLgDcvsOACADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQ -Ags= +A0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7MCAgR/AWQAIAEhAi +AAIAAQACIF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAAIAJBAWoFIANB +gBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTSADQYCwA09xBH +8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZBP3FBgAFyOgAA +IAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDHZB4AFyOgAAIA +JBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEBaiEEDAELCyAC +IAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIAJBAWohAgwCCy +AEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkECaiECDAILIARB +8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w4AIANBAWohAy +ACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFqLQAAQT9xQQx0 +ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3FBgLgDcvsOAC +ADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQAgs= `; diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts index 626e2578..5bab871f 100644 --- a/src/utils/utf8-wasm.ts +++ b/src/utils/utf8-wasm.ts @@ -147,11 +147,6 @@ export function utf8EncodeWasm(str: string, output: Uint8Array, outputOffset: nu * Uses GC arrays with fromCharCodeArray for efficient string creation. */ export function utf8DecodeWasm(bytes: Uint8Array, inputOffset: number, byteLength: number): string { - // Handle empty input - if (byteLength === 0) { - return ""; - } - // Ensure wasm memory is large enough for UTF-8 input const requiredPages = Math.ceil(byteLength / 65536); const currentPages = wasmInstance!.memory.buffer.byteLength / 65536; diff --git a/wasm/utf8.wat b/wasm/utf8.wat index 75c7aaf3..f882f0ed 100644 --- a/wasm/utf8.wat +++ b/wasm/utf8.wat @@ -84,10 +84,6 @@ (local.set $len (call $str_length (local.get $str))) (local.set $pos (local.get $offset)) - ;; Handle empty string - (if (i32.eqz (local.get $len)) - (then (return (i32.const 0)))) - ;; Allocate array and copy all char codes at once (local.set $arr (array.new $i16_array (i32.const 0) (local.get $len))) (drop (call $str_into_array (local.get $str) (local.get $arr) (i32.const 0))) From 5ff999d138d21600791a0599a021edf3bd5aa33a Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 23:04:25 +0900 Subject: [PATCH 08/13] update doc --- wasm/README.md | 325 ++++--------------------------------------------- 1 file changed, 24 insertions(+), 301 deletions(-) diff --git a/wasm/README.md b/wasm/README.md index be8935e9..4372876f 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -1,4 +1,4 @@ -# WebAssembly String Processing Plan +# WebAssembly UTF-8 String Processing ## Background @@ -14,321 +14,44 @@ The main issues were: ### What Changed in 2025 -**js-string-builtins** (WebAssembly 3.0, September 2025) fundamentally changes the equation: +**js-string-builtins** (WebAssembly 3.0) fundamentally changes the equation: -- Direct import of JS string operations (`length`, `charCodeAt`, `substring`, etc.) from `wasm:js-string` +- Direct import of JS string operations from `wasm:js-string` - No glue code overhead - operations can be inlined by the engine -- No memory copying at boundaries when consuming JS strings -- Strings stay in JS representation (UTF-16) - no UTF-8/UTF-16 conversion +- Uses WASM GC arrays with `intoCharCodeArray`/`fromCharCodeArray` for bulk operations -Browser/runtime support: -- Chrome 131+ (enabled by default) -- Firefox 134+ -- Safari: TBD (expressed openness) -- Node.js 24+ (V8 13.6+, enabled by default) -- Node.js 22-23: `--experimental-wasm-imported-strings` flag required +## Building -## Proposal: Hand-written WAT - -### Why WAT over Rust/wasm-bindgen - -| Aspect | Hand-written WAT | Rust + wasm-bindgen | -|--------|------------------|---------------------| -| Overhead | Zero - direct builtins | Glue code overhead | -| Binary size | Minimal (~1-2KB) | Larger (~10KB+) | -| Dependencies | None (just wat2wasm) | Rust toolchain, wasm-pack | -| Complexity | Simple for small scope | Overkill for 3 functions | -| js-string-builtins | Direct imports | Indirect, still evolving | -| Contributor barrier | Low (WAT is simple) | Higher (Rust knowledge) | - -For our limited scope (UTF-8 encode/decode), hand-written WAT is ideal. - -### What to Implement in Wasm - -1. **UTF-8 byte length counting** (`utf8Count`) - - Iterate string via `charCodeAt`, calculate byte length - -2. **UTF-8 encoding** (`utf8Encode`) - - Read chars via `charCodeAt`, write UTF-8 bytes to linear memory - -3. **UTF-8 decoding** (`utf8Decode`) - - Read UTF-8 bytes from memory, build string via `fromCharCode`/`fromCodePoint` - -### Available js-string-builtins - -From `wasm:js-string`: -- `length` - get string length -- `charCodeAt` - get UTF-16 code unit at index -- `codePointAt` - get Unicode code point at index (handles surrogates) -- `fromCharCode` - create single-char string from code unit -- `fromCodePoint` - create single-char string from code point -- `concat` - concatenate strings -- `substring` - extract substring -- `equals` - compare strings - -## Implementation Plan - -### Phase 1: Project Setup - -``` -msgpack-javascript/ -β”œβ”€β”€ wasm/ -β”‚ β”œβ”€β”€ utf8.wat # hand-written WAT source -β”‚ └── build.sh # wat2wasm + base64 generation -β”œβ”€β”€ src/ -β”‚ └── utils/ -β”‚ β”œβ”€β”€ utf8.ts # existing pure JS -β”‚ β”œβ”€β”€ utf8-wasm.ts # wasm loader + integration -β”‚ └── utf8-wasm-binary.ts # auto-generated base64 wasm -``` - -### Phase 2: WAT Implementation - -```wat -;; wasm/utf8.wat -(module - ;; Import js-string builtins - ;; Note: string parameters use externref, string returns use (ref extern) - (import "wasm:js-string" "length" - (func $str_length (param externref) (result i32))) - (import "wasm:js-string" "charCodeAt" - (func $str_charCodeAt (param externref i32) (result i32))) - (import "wasm:js-string" "fromCharCode" - (func $str_fromCharCode (param i32) (result (ref extern)))) - (import "wasm:js-string" "concat" - (func $str_concat (param externref externref) (result (ref extern)))) - - ;; Linear memory for UTF-8 bytes (exported for JS access) - (memory (export "memory") 1) - - ;; Count UTF-8 byte length of a JS string - (func (export "utf8Count") (param $str externref) (result i32) - (local $i i32) - (local $len i32) - (local $byteLen i32) - (local $code i32) - - (local.set $len (call $str_length (local.get $str))) - - (block $break - (loop $continue - (br_if $break (i32.ge_u (local.get $i) (local.get $len))) - - (local.set $code - (call $str_charCodeAt (local.get $str) (local.get $i))) - - ;; Count bytes based on code point range - (if (i32.lt_u (local.get $code) (i32.const 0x80)) - (then - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 1)))) - (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) - (then - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 2)))) - (else (if (i32.and - (i32.ge_u (local.get $code) (i32.const 0xD800)) - (i32.le_u (local.get $code) (i32.const 0xDBFF))) - ;; High surrogate - 4 bytes total, skip low surrogate - (then - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 4))) - (local.set $i (i32.add (local.get $i) (i32.const 1)))) - (else - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 3))))))))) - - (local.set $i (i32.add (local.get $i) (i32.const 1))) - (br $continue))) - - (local.get $byteLen)) - - ;; Encode JS string to UTF-8 bytes at offset, returns bytes written - (func (export "utf8Encode") (param $str externref) (param $offset i32) (result i32) - ;; Similar loop: charCodeAt -> encode -> store to memory - (local $i i32) - (local $len i32) - (local $pos i32) - (local $code i32) - - (local.set $len (call $str_length (local.get $str))) - (local.set $pos (local.get $offset)) - - (block $break - (loop $continue - (br_if $break (i32.ge_u (local.get $i) (local.get $len))) - - (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) - - ;; 1-byte (ASCII) - (if (i32.lt_u (local.get $code) (i32.const 0x80)) - (then - (i32.store8 (local.get $pos) (local.get $code)) - (local.set $pos (i32.add (local.get $pos) (i32.const 1)))) - (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) - ;; 2-byte - (then - (i32.store8 (local.get $pos) - (i32.or (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0xC0))) - (i32.store8 (i32.add (local.get $pos) (i32.const 1)) - (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) - (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) - ;; 3-byte or 4-byte (surrogate pair) - (else - ;; TODO: handle surrogates for 4-byte - (i32.store8 (local.get $pos) - (i32.or (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0xE0))) - (i32.store8 (i32.add (local.get $pos) (i32.const 1)) - (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) - (i32.store8 (i32.add (local.get $pos) (i32.const 2)) - (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) - (local.set $pos (i32.add (local.get $pos) (i32.const 3))))))) - - (local.set $i (i32.add (local.get $i) (i32.const 1))) - (br $continue))) - - (i32.sub (local.get $pos) (local.get $offset))) - - ;; Decode UTF-8 bytes from memory to JS string - (func (export "utf8Decode") (param $offset i32) (param $length i32) (result externref) - ;; Build string by reading bytes, decoding, calling fromCharCode + concat - ;; ... implementation - (call $str_fromCharCode (i32.const 0))) ;; placeholder -) -``` - -### Phase 3: Build Script +Requires [Binaryen](https://github.com/WebAssembly/binaryen) (`brew install binaryen`): ```bash -#!/bin/bash -# wasm/build.sh -# Requires: binaryen (brew install binaryen) - -wasm-as utf8.wat -o utf8.wasm --enable-reference-types --enable-gc - -# Generate base64-encoded TypeScript module -echo "// Auto-generated - do not edit" > ../src/utils/utf8-wasm-binary.ts -echo "export const wasmBinary = \"$(base64 -i utf8.wasm)\";" >> ../src/utils/utf8-wasm-binary.ts +./build.sh ``` -### Phase 4: TypeScript Integration - -```typescript -// src/utils/utf8-wasm-binary.ts (auto-generated) -export const wasmBinary = "AGFzbQEAAAA..."; // base64-encoded wasm - -// src/utils/utf8-wasm.ts -import { utf8Count as utf8CountJs } from "./utf8.js"; -import { wasmBinary } from "./utf8-wasm-binary.js"; - -interface WasmExports { - memory: WebAssembly.Memory; - utf8Count(str: string): number; - utf8Encode(str: string, offset: number): number; - utf8Decode(offset: number, length: number): string; -} - -let wasm: WasmExports | null = null; - -function base64ToBytes(base64: string): Uint8Array { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) { - bytes[i] = binary.charCodeAt(i); - } - return bytes; -} - -// Polyfill for js-string-builtins (used when native builtins unavailable) -const jsStringPolyfill = { - "wasm:js-string": { - length: (s: string) => s.length, - charCodeAt: (s: string, i: number) => s.charCodeAt(i), - codePointAt: (s: string, i: number) => s.codePointAt(i), - fromCharCode: (code: number) => String.fromCharCode(code), - fromCodePoint: (code: number) => String.fromCodePoint(code), - concat: (a: string, b: string) => a + b, - substring: (s: string, start: number, end: number) => s.substring(start, end), - equals: (a: string, b: string) => a === b, - }, -}; - -// Synchronous initialization -function initWasm(): boolean { - if (wasm) return true; - - try { - const bytes = base64ToBytes(wasmBinary); - // Try with builtins first (native support) - // If builtins not supported, option is ignored and polyfill is used - const module = new WebAssembly.Module(bytes, { builtins: ["js-string"] }); - const instance = new WebAssembly.Instance(module, jsStringPolyfill); - wasm = instance.exports as WasmExports; - return true; - } catch { - return false; // Fallback to pure JS (utf8CountJs, etc.) - } -} - -// Try init at module load -const wasmAvailable = initWasm(); - -export function utf8Count(str: string): number { - return wasm ? wasm.utf8Count(str) : utf8CountJs(str); -} -``` - -**Progressive enhancement:** -- Native builtins β†’ engine ignores import object, uses optimized builtins -- No native builtins β†’ engine uses polyfill from import object -- Wasm fails entirely β†’ falls back to pure JS implementation - -**Benefits of base64 inline:** -- No async initialization needed - sync `new WebAssembly.Module()` -- No fetch/network request - works in all environments -- Single file distribution - no separate .wasm asset -- Bundle size: ~1.3x wasm size (base64 overhead), but gzip compresses well - -## Compatibility Matrix - -| Environment | Native builtins | Wasm + polyfill | Pure JS fallback | -|-------------|-----------------|-----------------|------------------| -| Chrome 131+ | Yes | - | - | -| Firefox 134+ | Yes | - | - | -| Safari 18+ | TBD | Yes | - | -| Node.js 24+ | Yes (V8 13.6+) | - | - | -| Node.js 22-23 | Flag required | Yes | - | -| Deno | TBD | Yes | - | -| Older browsers | No | Yes | - | -| No Wasm support | - | - | Yes | - -Three-tier fallback: -1. **Native builtins** - best performance (engine-optimized) -2. **Wasm + polyfill** - good performance (wasm logic, JS string ops) -3. **Pure JS** - baseline (current implementation) - -## Benchmarking Strategy - -1. Reuse existing benchmarks: - - `benchmark/encode-string.ts` - - `benchmark/decode-string.ts` +This compiles `utf8.wat` and generates `src/utils/utf8-wasm-binary.ts` with the base64-encoded binary. -2. Add Wasm variants and compare across string sizes: - - Short strings (< 50 bytes): likely JS faster due to call overhead - - Medium strings (50-1000 bytes): Wasm should win - - Large strings (> 1000 bytes): TextEncoder/TextDecoder still optimal +## Runtime Requirements -## Success Criteria +| Environment | Support | +|-------------|---------| +| Node.js 24+ | Native (V8 13.6+) | +| Node.js 22-23 | `--experimental-wasm-imported-strings` flag | +| Chrome 131+ | Native | +| Firefox 134+ | Native | +| Safari | TBD | +| Older/unsupported | Falls back to pure JS | -1. **Performance**: >= 1.5x speedup for medium strings (50-1000 bytes) -2. **Bundle size**: Wasm binary < 2KB (~2.7KB as base64, compresses well with gzip) -3. **Compatibility**: Zero breakage with fallback to pure JS -4. **Maintainability**: Simple WAT, easy to understand +## Architecture -## Decisions +Three-tier dispatch based on string/byte length: -- **Node.js**: js-string-builtins enabled by default in Node.js 24+ (V8 13.6+). For Node.js 22-23, use `--experimental-wasm-imported-strings` flag. +| Length | Method | Reason | +|--------|--------|--------| +| ≀ 50 | Pure JS | Lowest call overhead | +| 51-1000 | WASM | Optimal for medium strings | +| > 1000 | TextEncoder/TextDecoder | SIMD-optimized for bulk | ## References - [js-string-builtins proposal](https://github.com/WebAssembly/js-string-builtins) - [MDN: WebAssembly JavaScript builtins](https://developer.mozilla.org/en-US/docs/WebAssembly/Guides/JavaScript_builtins) -- [WebAssembly 3.0 announcement](https://webassembly.org/news/2025-09-17-wasm-3.0/) -- [Previous PR #26](https://github.com/msgpack/msgpack-javascript/pull/26) -- [Removal PR #95](https://github.com/msgpack/msgpack-javascript/pull/95) From 29f5afaea943290897770502e8cd5d9138d732f9 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sat, 27 Dec 2025 23:39:36 +0900 Subject: [PATCH 09/13] optimize utf8count wasm --- benchmark/count-utf8.ts | 57 +++++++++++++++++++++++++++++++++++ src/utils/utf8-wasm-binary.ts | 36 +++++++++++----------- wasm/utf8.wat | 15 +++------ 3 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 benchmark/count-utf8.ts diff --git a/benchmark/count-utf8.ts b/benchmark/count-utf8.ts new file mode 100644 index 00000000..babfbc21 --- /dev/null +++ b/benchmark/count-utf8.ts @@ -0,0 +1,57 @@ +/* eslint-disable no-console */ +import { utf8CountJs, WASM_AVAILABLE } from "../src/utils/utf8.ts"; +import { getWasmError, utf8CountWasm } from "../src/utils/utf8-wasm.ts"; + +// @ts-ignore +import Benchmark from "benchmark"; + +// description +console.log("utf8CountJs - pure JS implementation"); +console.log("utf8CountWasm - WebAssembly implementation"); + +// Show wasm status +console.log("=".repeat(60)); +console.log("WebAssembly Status:"); +console.log(` WASM_AVAILABLE: ${WASM_AVAILABLE}`); +if (WASM_AVAILABLE) { + console.log(" js-string-builtins: enabled"); +} else { + const error = getWasmError(); + console.log(` Error: ${error?.message || "unknown"}`); + if (error?.message?.includes("js-string") || error?.message?.includes("builtin")) { + console.log("\n js-string-builtins is enabled by default in Node.js 24+ (V8 13.6+)."); + console.log(" For older versions, run with:"); + console.log(" node --experimental-wasm-imported-strings node_modules/.bin/ts-node benchmark/count-utf8.ts"); + } +} +console.log("=".repeat(60)); + +for (const baseStr of ["A", "あ", "🌏"]) { + const dataSet = [10, 30, 50, 100, 200, 500, 1000].map((n) => { + return baseStr.repeat(n); + }); + + for (const str of dataSet) { + const byteLength = utf8CountJs(str); + + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); + + const suite = new Benchmark.Suite(); + + suite.add("utf8CountJs", () => { + utf8CountJs(str); + }); + + if (WASM_AVAILABLE) { + suite.add("utf8CountWasm", () => { + utf8CountWasm(str); + }); + } + + suite.on("cycle", (event: any) => { + console.log(String(event.target)); + }); + + suite.run(); + } +} diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index 8e46f56b..cb6198d9 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -2,22 +2,22 @@ // Source: wasm/utf8.wat export const wasmBinary = ` -AGFzbQEAAAABNQhedwFgAW8Bf2ADb2QAfwF/YANkAH9/AWRvYAJvfwF/YAJ/ZAABf2ABfwFkAGADZA -B/fwFvAl8DDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nEWludG9DaGFyQ29k -ZUFycmF5AAIOd2FzbTpqcy1zdHJpbmcRZnJvbUNoYXJDb2RlQXJyYXkAAwMGBQEEBQYHBQMBAAEHVA -YGbWVtb3J5AgAJdXRmOENvdW50AAMKdXRmOEVuY29kZQAEEXV0ZjhEZWNvZGVUb0FycmF5AAUKYWxs -b2NBcnJheQAGDWFycmF5VG9TdHJpbmcABwqQBgWEAQIEfwFkACAAEAAiBEUEQEEADwsgACAE+wcAIg -VBABABGgNAIAEgBE9FBEAgBSAB+w0AIgNBgAFJBH8gAkEBagUgA0GAEEkEfyACQQJqBSADQf+3A00g -A0GAsANPcQR/IAFBAWohASACQQRqBSACQQNqCwsLIQIgAUEBaiEBDAELCyACC7MCAgR/AWQAIAEhAi -AAIAAQACIF+wcAIgZBABABGgNAIAQgBU9FBEAgBiAE+w0AIgNBgAFJBH8gAiADOgAAIAJBAWoFIANB -gBBJBH8gAiADQQZ2QcABcjoAACACQQFqIANBP3FBgAFyOgAAIAJBAmoFIANB/7cDTSADQYCwA09xBH -8gAiADQQp0IAYgBEEBaiIE+w0AakGAuP8aayIDQRJ2QfABcjoAACACQQFqIANBDHZBP3FBgAFyOgAA -IAJBAmogA0EGdkE/cUGAAXI6AAAgAkEDaiADQT9xQYABcjoAACACQQRqBSACIANBDHZB4AFyOgAAIA -JBAWogA0EGdkE/cUGAAXI6AAAgAkECaiADQT9xQYABcjoAACACQQNqCwsLIQIgBEEBaiEEDAELCyAC -IAFrC78CAQN/A0AgACACSwRAIAItAAAiBEGAAXFFBEAgASADIAT7DgAgA0EBaiEDIAJBAWohAgwCCy -AEQeABcUHAAUYEQCABIAMgAkEBai0AAEE/cSAEQR9xQQZ0cvsOACADQQFqIQMgAkECaiECDAILIARB -8AFxQeABRgRAIAEgAyACQQJqLQAAQT9xIARBD3FBDHQgAkEBai0AAEE/cUEGdHJy+w4AIANBAWohAy -ACQQNqIQIMAgsgBEH4AXFB8AFGBEAgASADIAJBA2otAABBP3EgBEEHcUESdCACQQFqLQAAQT9xQQx0 -ciACQQJqLQAAQT9xQQZ0cnJBgIAEayIEQQp2QYCwA3L7DgAgASADQQFqIgMgBEH/B3FBgLgDcvsOAC -ADQQFqIQMgAkEEaiECDAIFIAJBAWohAgwCCwALCyADCwcAIAD7BwALCgAgACABIAIQAgs= +AGFzbQEAAAABNQhedwFgAW8Bf2ACb38Bf2ADb2QAfwF/YANkAH9/AWRvYAJ/ZAABf2ABfwFkAGADZA +B/fwFvAnsEDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nCmNoYXJDb2RlQXQA +Ag53YXNtOmpzLXN0cmluZxFpbnRvQ2hhckNvZGVBcnJheQADDndhc206anMtc3RyaW5nEWZyb21DaG +FyQ29kZUFycmF5AAQDBgUBAgUGBwUDAQABB1QGBm1lbW9yeQIACXV0ZjhDb3VudAAECnV0ZjhFbmNv +ZGUABRF1dGY4RGVjb2RlVG9BcnJheQAGCmFsbG9jQXJyYXkABw1hcnJheVRvU3RyaW5nAAgK9gUFaw +EEfyAAEAAhBANAIAEgBE9FBEAgACABEAEiA0GAAUkEfyACQQFqBSADQYAQSQR/IAJBAmoFIANB/7cD +TSADQYCwA09xBH8gAUEBaiEBIAJBBGoFIAJBA2oLCwshAiABQQFqIQEMAQsLIAILswICBH8BZAAgAS +ECIAAgABAAIgX7BwAiBkEAEAIaA0AgBCAFT0UEQCAGIAT7DQAiA0GAAUkEfyACIAM6AAAgAkEBagUg +A0GAEEkEfyACIANBBnZBwAFyOgAAIAJBAWogA0E/cUGAAXI6AAAgAkECagUgA0H/twNNIANBgLADT3 +EEfyACIANBCnQgBiAEQQFqIgT7DQBqQYC4/xprIgNBEnZB8AFyOgAAIAJBAWogA0EMdkE/cUGAAXI6 +AAAgAkECaiADQQZ2QT9xQYABcjoAACACQQNqIANBP3FBgAFyOgAAIAJBBGoFIAIgA0EMdkHgAXI6AA +AgAkEBaiADQQZ2QT9xQYABcjoAACACQQJqIANBP3FBgAFyOgAAIAJBA2oLCwshAiAEQQFqIQQMAQsL +IAIgAWsLvwIBA38DQCAAIAJLBEAgAi0AACIEQYABcUUEQCABIAMgBPsOACADQQFqIQMgAkEBaiECDA +ILIARB4AFxQcABRgRAIAEgAyACQQFqLQAAQT9xIARBH3FBBnRy+w4AIANBAWohAyACQQJqIQIMAgsg +BEHwAXFB4AFGBEAgASADIAJBAmotAABBP3EgBEEPcUEMdCACQQFqLQAAQT9xQQZ0cnL7DgAgA0EBai +EDIAJBA2ohAgwCCyAEQfgBcUHwAUYEQCABIAMgAkEDai0AAEE/cSAEQQdxQRJ0IAJBAWotAABBP3FB +DHRyIAJBAmotAABBP3FBBnRyckGAgARrIgRBCnZBgLADcvsOACABIANBAWoiAyAEQf8HcUGAuANy+w +4AIANBAWohAyACQQRqIQIMAgUgAkEBaiECDAILAAsLIAMLBwAgAPsHAAsKACAAIAEgAhADCw== `; diff --git a/wasm/utf8.wat b/wasm/utf8.wat index f882f0ed..2bfc8dcf 100644 --- a/wasm/utf8.wat +++ b/wasm/utf8.wat @@ -11,6 +11,8 @@ ;; Import js-string builtins (import "wasm:js-string" "length" (func $str_length (param externref) (result i32))) + (import "wasm:js-string" "charCodeAt" + (func $str_charCodeAt (param externref i32) (result i32))) (import "wasm:js-string" "intoCharCodeArray" (func $str_into_array (param externref (ref $i16_array) i32) (result i32))) (import "wasm:js-string" "fromCharCodeArray" @@ -20,30 +22,21 @@ (memory (export "memory") 1) ;; Count UTF-8 byte length of a JS string - ;; Uses GC array to get all char codes at once + ;; Uses charCodeAt directly to avoid array allocation overhead (func (export "utf8Count") (param $str externref) (result i32) (local $len i32) - (local $arr (ref $i16_array)) (local $i i32) (local $byteLen i32) (local $code i32) (local.set $len (call $str_length (local.get $str))) - ;; Handle empty string - (if (i32.eqz (local.get $len)) - (then (return (i32.const 0)))) - - ;; Allocate array and copy string chars - (local.set $arr (array.new $i16_array (i32.const 0) (local.get $len))) - (drop (call $str_into_array (local.get $str) (local.get $arr) (i32.const 0))) - ;; Count UTF-8 bytes (block $break (loop $continue (br_if $break (i32.ge_u (local.get $i) (local.get $len))) - (local.set $code (array.get_u $i16_array (local.get $arr) (local.get $i))) + (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) ;; 1-byte: 0x00-0x7F (if (i32.lt_u (local.get $code) (i32.const 0x80)) From 566bd0047d70f2c587ac036399f08c02b090132c Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sun, 28 Dec 2025 11:40:37 +0900 Subject: [PATCH 10/13] simplify the code --- src/utils/utf8-wasm.ts | 40 ++++++++++++++++++++-------------------- tsconfig.json | 41 ++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/utils/utf8-wasm.ts b/src/utils/utf8-wasm.ts index 5bab871f..89a029ce 100644 --- a/src/utils/utf8-wasm.ts +++ b/src/utils/utf8-wasm.ts @@ -11,26 +11,20 @@ import { wasmBinary } from "./utf8-wasm-binary.ts"; -// Check environment variable for wasm mode -declare const process: { env?: Record } | undefined; - function getWasmMode(): "force" | "never" | "auto" { - try { - if (process?.env) { - const mode = process.env["MSGPACK_WASM"]; - if (mode) { - switch (mode.toLowerCase()) { - case "force": - return "force"; - case "never": - return "never"; - default: - return "auto"; - } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (typeof process !== "undefined" && process.env) { + const mode = process.env["MSGPACK_WASM"]; + if (mode) { + switch (mode.toLowerCase()) { + case "force": + return "force"; + case "never": + return "never"; + default: + return "auto"; } } - } catch { - // process may not be defined in browser } return "auto"; } @@ -53,7 +47,15 @@ let wasmInstance: WasmExports | null = null; let wasmInitError: Error | null = null; function base64ToBytes(base64: string): Uint8Array { - if (typeof atob === "function") { + // @ts-expect-error - fromBase64 is not yet supported in TypeScript + if (Uint8Array.fromBase64) { + // @ts-expect-error - fromBase64 is not yet supported in TypeScript + return Uint8Array.fromBase64(base64); + } else if (typeof Buffer !== "undefined") { + // Node.js + return new Uint8Array(Buffer.from(base64, "base64")); + } else { + // Legacy fallback const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) { @@ -61,8 +63,6 @@ function base64ToBytes(base64: string): Uint8Array { } return bytes; } - // Node.js fallback - return new Uint8Array(Buffer.from(base64, "base64")); } function tryInitializeWasmInstance(): void { diff --git a/tsconfig.json b/tsconfig.json index a4cacbca..fc7ca8fb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,29 +1,31 @@ { "compilerOptions": { /* Basic Options */ - "target": "ES2020", /* the baseline */ - "module": "CommonJS", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ - "lib": ["ES2024", "DOM"], /* Specify library files to be included in the compilation. */ + "target": "es2020", /* the baseline */ + "module": "CommonJS", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "lib": [ + "esnext", + "dom" + ], /* Specify library files to be included in the compilation. */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ - "sourceMap": true, /* Generates corresponding '.map' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ // "outFile": "./", /* Concatenate and emit output to single file. */ - "outDir": "./build", /* Redirect output structure to the directory. */ + "outDir": "./build", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ // "composite": true, /* Enable project compilation */ - "incremental": true, /* Enable incremental compilation */ + "incremental": true, /* Enable incremental compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ // "removeComments": true, /* Do not emit comments to output. */ // "noEmit": true, /* Do not emit outputs. */ - "importHelpers": false, /* Import emit helpers from 'tslib'. */ + "importHelpers": false, /* Import emit helpers from 'tslib'. */ // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ - /* Strict Type-Checking Options */ - "strict": true, /* Enable all strict type-checking options. */ + "strict": true, /* Enable all strict type-checking options. */ // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ // "strictNullChecks": true, /* Enable strict null checks. */ // "strictFunctionTypes": true, /* Enable strict checking of function types. */ @@ -31,21 +33,19 @@ // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ - /* Additional Checks */ // "noUnusedLocals": true, /* Report errors on unused locals. */ // "noUnusedParameters": true, /* Report errors on unused parameters. */ - "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ - "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ "noUncheckedIndexedAccess": true, "noPropertyAccessFromIndexSignature": true, "noImplicitOverride": true, "verbatimModuleSyntax": false, "allowImportingTsExtensions": true, "noEmit": true, - /* Module Resolution Options */ - "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ // "paths": { // "@msgpack/msgpack": ["./src"] // }, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ @@ -53,23 +53,26 @@ // "typeRoots": [], /* List of folders to include type definitions from. */ // "types": [], /* Type declaration files to be included in compilation. */ // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ - "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ "resolveJsonModule": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true - // "erasableSyntaxOnly": true - /* Source Map Options */ // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ - /* Experimental Options */ // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ }, - "exclude": ["example", "benchmark", "test/bun*", "test/deno*", "mod.ts"] + "exclude": [ + "example", + "benchmark", + "test/bun*", + "test/deno*", + "mod.ts" + ] } From 911ed74855dead006eb699b110a58a313a47afe0 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sun, 28 Dec 2025 12:10:18 +0900 Subject: [PATCH 11/13] add two skills about binary and wasm operations --- .claude/skills/typed-arrays/SKILL.md | 115 +++++++++++++++++++++++++++ .claude/skills/wasm/SKILL.md | 101 +++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 .claude/skills/typed-arrays/SKILL.md create mode 100644 .claude/skills/wasm/SKILL.md diff --git a/.claude/skills/typed-arrays/SKILL.md b/.claude/skills/typed-arrays/SKILL.md new file mode 100644 index 00000000..89ceb8fa --- /dev/null +++ b/.claude/skills/typed-arrays/SKILL.md @@ -0,0 +1,115 @@ +--- +name: typed-arrays +description: | + Modern TypedArray and ArrayBuffer features including resizable buffers, + transfer operations, Float16Array, and Uint8Array base64/hex encoding. +compatibility: Node.js 20+ and all the modern browsers +--- + +# Modern Typed Arrays + +## ES2023: Change Array by Copy + +Immutable operations returning new arrays: + +```typescript +const arr = new Uint8Array([3, 1, 2]); + +arr.toReversed(); // Uint8Array [2, 1, 3] +arr.toSorted((a, b) => a - b); // Uint8Array [1, 2, 3] +arr.with(0, 99); // Uint8Array [99, 1, 2] +``` + +## ES2023: findLast / findLastIndex + +```typescript +const arr = new Uint8Array([1, 2, 3, 2, 1]); + +arr.findLast(x => x === 2); // 2 +arr.findLastIndex(x => x === 2); // 3 +``` + +## ES2024: Resizable ArrayBuffer + +```typescript +const buffer = new ArrayBuffer(16, { maxByteLength: 1024 }); + +buffer.resizable; // true +buffer.maxByteLength; // 1024 +buffer.resize(64); // grow +buffer.resize(8); // shrink +``` + +### Growable SharedArrayBuffer + +```typescript +const shared = new SharedArrayBuffer(16, { maxByteLength: 1024 }); +shared.growable; // true +shared.grow(64); // can only grow, not shrink +``` + +### TypedArray tracks resizable buffer + +```typescript +const buffer = new ArrayBuffer(16, { maxByteLength: 64 }); +const view = new Uint8Array(buffer); +view.length; // 16 +buffer.resize(32); +view.length; // 32 (auto-tracks) +``` + +## ES2024: ArrayBuffer Transfer + +```typescript +const buffer = new ArrayBuffer(16); +const arr = new Uint8Array(buffer); +arr[0] = 42; + +const newBuffer = buffer.transfer(); // zero-copy transfer +buffer.detached; // true +newBuffer.byteLength; // 16 + +// Transfer with resize +const grown = buffer.transfer(64); + +// Convert resizable to fixed +const fixed = resizable.transferToFixedLength(); +``` + +## ES2025: Float16Array + +```typescript +const f16 = new Float16Array(4); +const f16arr = Float16Array.of(1.5, 2.5, 3.5); + +Float16Array.BYTES_PER_ELEMENT; // 2 +// Range: Β±65504 (max), Β±6.1e-5 (min positive) +``` + +### DataView Float16 + +```typescript +const view = new DataView(buffer); +view.setFloat16(0, 3.14, true); // little-endian +view.getFloat16(0, true); // β‰ˆ3.140625 +``` + +## ES2026: Uint8Array Base64 + +Not yet in Node.js v24. + +### Base64 + +```typescript +const bytes = new Uint8Array([72, 101, 108, 108, 111]); + +bytes.toBase64(); // "SGVsbG8=" +bytes.toBase64({ alphabet: "base64url" }); // URL-safe +bytes.toBase64({ omitPadding: true }); // no trailing = + +Uint8Array.fromBase64("SGVsbG8="); +Uint8Array.fromBase64("SGVsbG8", { alphabet: "base64url" }); + +// Write to existing buffer +const { read, written } = target.setFromBase64("SGVsbG8="); +``` diff --git a/.claude/skills/wasm/SKILL.md b/.claude/skills/wasm/SKILL.md new file mode 100644 index 00000000..e7fa6a34 --- /dev/null +++ b/.claude/skills/wasm/SKILL.md @@ -0,0 +1,101 @@ +--- +name: wasm +description: | + Modern WebAssembly (WASM) development expertise covering WASM 3.0 features + and optimization techniques. Use this skill when working with WebAssembly + modules, optimizing WASM performance, or integrating WASM with JavaScript/TypeScript. +compatibility: WebAssembly v3.0 and later +--- + +# WebAssembly Development Skill + +## WAT Syntax + +Use **folded (S-expression) syntax** for readability: + +```wat +;; Folded syntax (preferred) +(i32.add (local.get $x) (local.get $y)) + +;; Flat syntax (avoid) +local.get $x +local.get $y +i32.add +``` + +## WebAssembly 3.0 Features + +### Memory64 (64-bit Address Space) +- Memories and tables use `i64` as address type +- Expands addressable space from 4GB to 16 exabytes +- Syntax: `(memory i64 1)` instead of `(memory 1)` + +### Multiple Memories +```wat +(module + (memory $main 1) + (memory $scratch 1)) +``` + +### Tail Call Optimization +- Efficient recursion via `return_call` and `return_call_indirect` +- Prevents stack overflow for tail-recursive functions +```wat +(func $factorial (param $n i64) (param $acc i64) (result i64) + (if (result i64) (i64.eqz (local.get $n)) + (then (local.get $acc)) + (else (return_call $factorial + (i64.sub (local.get $n) (i64.const 1)) + (i64.mul (local.get $n) (local.get $acc)))))) +``` + +### Exception Handling +- Native try/catch/throw semantics +- Interoperates with JavaScript exceptions +```wat +(tag $error (param i32)) +(func $may_throw + (throw $error (i32.const 42))) +``` + +### Relaxed SIMD +- Hardware-dependent SIMD optimizations beyond fixed-width 128-bit +- `i8x16.relaxed_swizzle`, `f32x4.relaxed_madd`, etc. + +### SIMD Example +```wat +;; Process 16 bytes at a time +(v128.store (local.get $dst) + (i8x16.add + (v128.load (local.get $src1)) + (v128.load (local.get $src2)))) +``` + +## Toolchain (Binaryen) + +| Task | Command | +|------|---------| +| Assemble WAT to WASM | `wasm-as module.wat -o module.wasm` | +| Disassemble WASM to WAT | `wasm-dis module.wasm -o module.wat` | +| Optimize for size | `wasm-opt -Oz in.wasm -o out.wasm` | +| Optimize for speed | `wasm-opt -O3 in.wasm -o out.wasm` | + +## JavaScript/TypeScript Integration + +### Instantiation +```typescript +const module = await WebAssembly.compileStreaming(fetch('module.wasm')); +const instance = await WebAssembly.instantiate(module, imports); +``` + +### Memory Access +```typescript +const memory = new WebAssembly.Memory({ initial: 1, maximum: 100 }); +const buffer = new Uint8Array(instance.exports.memory.buffer); +buffer.set(data, offset); +``` + +## Resources + +- [WebAssembly Specification](https://webassembly.github.io/spec/) +- [Binaryen](https://github.com/WebAssembly/binaryen) From 84e8dc7f2869ea3789d5cc8a9e0ac400af824a66 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sun, 28 Dec 2025 12:18:57 +0900 Subject: [PATCH 12/13] tweaks --- .claude/skills/wasm/SKILL.md | 40 ++--- src/utils/utf8-wasm-binary.ts | 14 +- wasm/utf8.wat | 277 ++++++++++++++++------------------ 3 files changed, 158 insertions(+), 173 deletions(-) diff --git a/.claude/skills/wasm/SKILL.md b/.claude/skills/wasm/SKILL.md index e7fa6a34..a21d0863 100644 --- a/.claude/skills/wasm/SKILL.md +++ b/.claude/skills/wasm/SKILL.md @@ -1,9 +1,9 @@ --- name: wasm description: | - Modern WebAssembly (WASM) development expertise covering WASM 3.0 features + Modern WebAssembly (Wasm) development expertise covering modern Wasm features and optimization techniques. Use this skill when working with WebAssembly - modules, optimizing WASM performance, or integrating WASM with JavaScript/TypeScript. + modules or optimizing Wasm performance. compatibility: WebAssembly v3.0 and later --- @@ -23,7 +23,7 @@ local.get $y i32.add ``` -## WebAssembly 3.0 Features +## WebAssembly Features ### Memory64 (64-bit Address Space) - Memories and tables use `i64` as address type @@ -62,6 +62,21 @@ i32.add - Hardware-dependent SIMD optimizations beyond fixed-width 128-bit - `i8x16.relaxed_swizzle`, `f32x4.relaxed_madd`, etc. +### WasmGC +- Native garbage-collected types: `struct`, `array` +- Instructions: `array.new`, `array.get`, `array.set`, `struct.new`, `struct.get` +- Reference types: `(ref $type)`, `(ref null $type)` + +### externref +- Opaque reference to host (JS) objects +- Cannot be inspected or modified in Wasm, only passed around +- Used with js-string-builtins for efficient string handling + +### js-string-builtins +- Import `"wasm:js-string"` for direct JS string operations +- Functions: `length`, `charCodeAt`, `fromCharCodeArray`, `intoCharCodeArray` +- Avoids costly JS↔Wasm boundary crossings for string processing + ### SIMD Example ```wat ;; Process 16 bytes at a time @@ -75,26 +90,11 @@ i32.add | Task | Command | |------|---------| -| Assemble WAT to WASM | `wasm-as module.wat -o module.wasm` | -| Disassemble WASM to WAT | `wasm-dis module.wasm -o module.wat` | +| Assemble WAT to Wasm | `wasm-as module.wat -o module.wasm` | +| Disassemble Wasm to WAT | `wasm-dis module.wasm -o module.wat` | | Optimize for size | `wasm-opt -Oz in.wasm -o out.wasm` | | Optimize for speed | `wasm-opt -O3 in.wasm -o out.wasm` | -## JavaScript/TypeScript Integration - -### Instantiation -```typescript -const module = await WebAssembly.compileStreaming(fetch('module.wasm')); -const instance = await WebAssembly.instantiate(module, imports); -``` - -### Memory Access -```typescript -const memory = new WebAssembly.Memory({ initial: 1, maximum: 100 }); -const buffer = new Uint8Array(instance.exports.memory.buffer); -buffer.set(data, offset); -``` - ## Resources - [WebAssembly Specification](https://webassembly.github.io/spec/) diff --git a/src/utils/utf8-wasm-binary.ts b/src/utils/utf8-wasm-binary.ts index cb6198d9..8ade6c5c 100644 --- a/src/utils/utf8-wasm-binary.ts +++ b/src/utils/utf8-wasm-binary.ts @@ -6,7 +6,7 @@ AGFzbQEAAAABNQhedwFgAW8Bf2ACb38Bf2ADb2QAfwF/YANkAH9/AWRvYAJ/ZAABf2ABfwFkAGADZA B/fwFvAnsEDndhc206anMtc3RyaW5nBmxlbmd0aAABDndhc206anMtc3RyaW5nCmNoYXJDb2RlQXQA Ag53YXNtOmpzLXN0cmluZxFpbnRvQ2hhckNvZGVBcnJheQADDndhc206anMtc3RyaW5nEWZyb21DaG FyQ29kZUFycmF5AAQDBgUBAgUGBwUDAQABB1QGBm1lbW9yeQIACXV0ZjhDb3VudAAECnV0ZjhFbmNv -ZGUABRF1dGY4RGVjb2RlVG9BcnJheQAGCmFsbG9jQXJyYXkABw1hcnJheVRvU3RyaW5nAAgK9gUFaw +ZGUABRF1dGY4RGVjb2RlVG9BcnJheQAGCmFsbG9jQXJyYXkABw1hcnJheVRvU3RyaW5nAAgK6AUFaw EEfyAAEAAhBANAIAEgBE9FBEAgACABEAEiA0GAAUkEfyACQQFqBSADQYAQSQR/IAJBAmoFIANB/7cD TSADQYCwA09xBH8gAUEBaiEBIAJBBGoFIAJBA2oLCwshAiABQQFqIQEMAQsLIAILswICBH8BZAAgAS ECIAAgABAAIgX7BwAiBkEAEAIaA0AgBCAFT0UEQCAGIAT7DQAiA0GAAUkEfyACIAM6AAAgAkEBagUg @@ -14,10 +14,10 @@ A0GAEEkEfyACIANBBnZBwAFyOgAAIAJBAWogA0E/cUGAAXI6AAAgAkECagUgA0H/twNNIANBgLADT3 EEfyACIANBCnQgBiAEQQFqIgT7DQBqQYC4/xprIgNBEnZB8AFyOgAAIAJBAWogA0EMdkE/cUGAAXI6 AAAgAkECaiADQQZ2QT9xQYABcjoAACACQQNqIANBP3FBgAFyOgAAIAJBBGoFIAIgA0EMdkHgAXI6AA AgAkEBaiADQQZ2QT9xQYABcjoAACACQQJqIANBP3FBgAFyOgAAIAJBA2oLCwshAiAEQQFqIQQMAQsL -IAIgAWsLvwIBA38DQCAAIAJLBEAgAi0AACIEQYABcUUEQCABIAMgBPsOACADQQFqIQMgAkEBaiECDA -ILIARB4AFxQcABRgRAIAEgAyACQQFqLQAAQT9xIARBH3FBBnRy+w4AIANBAWohAyACQQJqIQIMAgsg -BEHwAXFB4AFGBEAgASADIAJBAmotAABBP3EgBEEPcUEMdCACQQFqLQAAQT9xQQZ0cnL7DgAgA0EBai -EDIAJBA2ohAgwCCyAEQfgBcUHwAUYEQCABIAMgAkEDai0AAEE/cSAEQQdxQRJ0IAJBAWotAABBP3FB -DHRyIAJBAmotAABBP3FBBnRyckGAgARrIgRBCnZBgLADcvsOACABIANBAWoiAyAEQf8HcUGAuANy+w -4AIANBAWohAyACQQRqIQIMAgUgAkEBaiECDAILAAsLIAMLBwAgAPsHAAsKACAAIAEgAhADCw== +IAIgAWsLsQIBA38DQCAAIANNRQRAIAMtAAAiBEGAAXEEfyAEQeABcUHAAUYEfyABIAIgA0EBai0AAE +E/cSAEQR9xQQZ0cvsOACACQQFqIQIgA0ECagUgBEHwAXFB4AFGBH8gASACIANBAmotAABBP3EgBEEP +cUEMdCADQQFqLQAAQT9xQQZ0cnL7DgAgAkEBaiECIANBA2oFIARB+AFxQfABRgR/IAEgAiADQQNqLQ +AAQT9xIARBB3FBEnQgA0EBai0AAEE/cUEMdHIgA0ECai0AAEE/cUEGdHJyQYCABGsiBEEKdkGAsANy ++w4AIAEgAkEBaiICIARB/wdxQYC4A3L7DgAgAkEBaiECIANBBGoFIANBAWoLCwsFIAEgAiAE+w4AIA +JBAWohAiADQQFqCyEDDAELCyACCwcAIAD7BwALCgAgACABIAIQAws= `; diff --git a/wasm/utf8.wat b/wasm/utf8.wat index 2bfc8dcf..9ca03079 100644 --- a/wasm/utf8.wat +++ b/wasm/utf8.wat @@ -1,11 +1,11 @@ ;; UTF-8 string processing using js-string-builtins with GC arrays ;; https://github.com/WebAssembly/js-string-builtins ;; -;; This implementation uses WASM GC arrays with intoCharCodeArray/fromCharCodeArray -;; for efficient bulk string operations instead of character-by-character processing. +;; Uses WASM GC arrays with intoCharCodeArray/fromCharCodeArray +;; for efficient bulk string operations. (module - ;; Define i16 array type for UTF-16 code units + ;; GC array type for UTF-16 code units (type $i16_array (array (mut i16))) ;; Import js-string builtins @@ -18,11 +18,10 @@ (import "wasm:js-string" "fromCharCodeArray" (func $str_from_array (param (ref $i16_array) i32 i32) (result (ref extern)))) - ;; Linear memory for UTF-8 bytes (64KB initial, exported for JS access) + ;; Linear memory for UTF-8 bytes (64KB initial) (memory (export "memory") 1) ;; Count UTF-8 byte length of a JS string - ;; Uses charCodeAt directly to avoid array allocation overhead (func (export "utf8Count") (param $str externref) (result i32) (local $len i32) (local $i i32) @@ -31,32 +30,32 @@ (local.set $len (call $str_length (local.get $str))) - ;; Count UTF-8 bytes (block $break (loop $continue (br_if $break (i32.ge_u (local.get $i) (local.get $len))) (local.set $code (call $str_charCodeAt (local.get $str) (local.get $i))) - ;; 1-byte: 0x00-0x7F (if (i32.lt_u (local.get $code) (i32.const 0x80)) (then + ;; 1-byte: 0x00-0x7F (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 1)))) - ;; 2-byte: 0x80-0x7FF - (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) - (then - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 2)))) - ;; Check for surrogate pair (high surrogate: 0xD800-0xDBFF) - (else (if (i32.and - (i32.ge_u (local.get $code) (i32.const 0xD800)) - (i32.le_u (local.get $code) (i32.const 0xDBFF))) - ;; 4-byte: surrogate pair, skip next char (low surrogate) + (else + (if (i32.lt_u (local.get $code) (i32.const 0x800)) (then - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 4))) - (local.set $i (i32.add (local.get $i) (i32.const 1)))) - ;; 3-byte: 0x800-0xFFFF (excluding surrogates) + ;; 2-byte: 0x80-0x7FF + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 2)))) (else - (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 3))))))))) + (if (i32.and + (i32.ge_u (local.get $code) (i32.const 0xD800)) + (i32.le_u (local.get $code) (i32.const 0xDBFF))) + (then + ;; 4-byte: surrogate pair, skip low surrogate + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 4))) + (local.set $i (i32.add (local.get $i) (i32.const 1)))) + (else + ;; 3-byte: 0x800-0xFFFF + (local.set $byteLen (i32.add (local.get $byteLen) (i32.const 3))))))))) (local.set $i (i32.add (local.get $i) (i32.const 1))) (br $continue))) @@ -65,7 +64,6 @@ ;; Encode JS string to UTF-8 bytes at offset in linear memory ;; Returns number of bytes written - ;; Uses intoCharCodeArray for bulk char code extraction (func (export "utf8Encode") (param $str externref) (param $offset i32) (result i32) (local $len i32) (local $arr (ref $i16_array)) @@ -77,89 +75,86 @@ (local.set $len (call $str_length (local.get $str))) (local.set $pos (local.get $offset)) - ;; Allocate array and copy all char codes at once + ;; Bulk copy all char codes into GC array (local.set $arr (array.new $i16_array (i32.const 0) (local.get $len))) (drop (call $str_into_array (local.get $str) (local.get $arr) (i32.const 0))) - ;; Encode to UTF-8 (block $break (loop $continue (br_if $break (i32.ge_u (local.get $i) (local.get $len))) (local.set $code (array.get_u $i16_array (local.get $arr) (local.get $i))) - ;; 1-byte: ASCII (0x00-0x7F) (if (i32.lt_u (local.get $code) (i32.const 0x80)) (then + ;; 1-byte: ASCII (i32.store8 (local.get $pos) (local.get $code)) (local.set $pos (i32.add (local.get $pos) (i32.const 1)))) - - ;; 2-byte: 0x80-0x7FF - (else (if (i32.lt_u (local.get $code) (i32.const 0x800)) - (then - (i32.store8 (local.get $pos) - (i32.or (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0xC0))) - (i32.store8 (i32.add (local.get $pos) (i32.const 1)) - (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) - (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) - - ;; Check for high surrogate (0xD800-0xDBFF) - (else (if (i32.and - (i32.ge_u (local.get $code) (i32.const 0xD800)) - (i32.le_u (local.get $code) (i32.const 0xDBFF))) - ;; 4-byte: surrogate pair + (else + (if (i32.lt_u (local.get $code) (i32.const 0x800)) (then - ;; Get low surrogate from array - (local.set $i (i32.add (local.get $i) (i32.const 1))) - (local.set $code2 (array.get_u $i16_array (local.get $arr) (local.get $i))) - ;; Calculate code point: ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000 - (local.set $code - (i32.add - (i32.add - (i32.shl - (i32.sub (local.get $code) (i32.const 0xD800)) - (i32.const 10)) - (i32.sub (local.get $code2) (i32.const 0xDC00))) - (i32.const 0x10000))) - ;; Encode 4-byte UTF-8 + ;; 2-byte: 110xxxxx 10xxxxxx (i32.store8 (local.get $pos) - (i32.or (i32.shr_u (local.get $code) (i32.const 18)) (i32.const 0xF0))) + (i32.or (i32.const 0xC0) (i32.shr_u (local.get $code) (i32.const 6)))) (i32.store8 (i32.add (local.get $pos) (i32.const 1)) - (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0x3F)) (i32.const 0x80))) - (i32.store8 (i32.add (local.get $pos) (i32.const 2)) - (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) - (i32.store8 (i32.add (local.get $pos) (i32.const 3)) - (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) - (local.set $pos (i32.add (local.get $pos) (i32.const 4)))) - - ;; 3-byte: 0x800-0xFFFF (excluding surrogates) + (i32.or (i32.const 0x80) (i32.and (local.get $code) (i32.const 0x3F)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) (else - (i32.store8 (local.get $pos) - (i32.or (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0xE0))) - (i32.store8 (i32.add (local.get $pos) (i32.const 1)) - (i32.or (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)) (i32.const 0x80))) - (i32.store8 (i32.add (local.get $pos) (i32.const 2)) - (i32.or (i32.and (local.get $code) (i32.const 0x3F)) (i32.const 0x80))) - (local.set $pos (i32.add (local.get $pos) (i32.const 3))))))))) + (if (i32.and + (i32.ge_u (local.get $code) (i32.const 0xD800)) + (i32.le_u (local.get $code) (i32.const 0xDBFF))) + (then + ;; 4-byte: surrogate pair + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (local.set $code2 (array.get_u $i16_array (local.get $arr) (local.get $i))) + ;; Decode: ((high - 0xD800) << 10) + (low - 0xDC00) + 0x10000 + (local.set $code + (i32.add + (i32.const 0x10000) + (i32.add + (i32.shl + (i32.sub (local.get $code) (i32.const 0xD800)) + (i32.const 10)) + (i32.sub (local.get $code2) (i32.const 0xDC00))))) + ;; 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + (i32.store8 (local.get $pos) + (i32.or (i32.const 0xF0) (i32.shr_u (local.get $code) (i32.const 18)))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.const 0x80) + (i32.and (i32.shr_u (local.get $code) (i32.const 12)) (i32.const 0x3F)))) + (i32.store8 (i32.add (local.get $pos) (i32.const 2)) + (i32.or (i32.const 0x80) + (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)))) + (i32.store8 (i32.add (local.get $pos) (i32.const 3)) + (i32.or (i32.const 0x80) (i32.and (local.get $code) (i32.const 0x3F)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 4)))) + (else + ;; 3-byte: 1110xxxx 10xxxxxx 10xxxxxx + (i32.store8 (local.get $pos) + (i32.or (i32.const 0xE0) (i32.shr_u (local.get $code) (i32.const 12)))) + (i32.store8 (i32.add (local.get $pos) (i32.const 1)) + (i32.or (i32.const 0x80) + (i32.and (i32.shr_u (local.get $code) (i32.const 6)) (i32.const 0x3F)))) + (i32.store8 (i32.add (local.get $pos) (i32.const 2)) + (i32.or (i32.const 0x80) (i32.and (local.get $code) (i32.const 0x3F)))) + (local.set $pos (i32.add (local.get $pos) (i32.const 3))))))))) (local.set $i (i32.add (local.get $i) (i32.const 1))) (br $continue))) (i32.sub (local.get $pos) (local.get $offset))) - ;; Decode UTF-8 bytes from linear memory to JS string - ;; Uses fromCharCodeArray for direct string creation - ;; Returns: (codeUnitsWritten << 16) | 0 for success, packed in i32 - ;; The actual string is returned via a separate export + ;; Decode UTF-8 bytes from linear memory to GC array + ;; Returns number of code units written (func (export "utf8DecodeToArray") (param $length i32) (param $arr (ref $i16_array)) (result i32) (local $pos i32) (local $end i32) (local $outIdx i32) - (local $byte1 i32) - (local $byte2 i32) - (local $byte3 i32) - (local $byte4 i32) - (local $codePoint i32) + (local $b1 i32) + (local $b2 i32) + (local $b3 i32) + (local $b4 i32) + (local $cp i32) (local.set $end (local.get $length)) @@ -167,84 +162,74 @@ (loop $continue (br_if $break (i32.ge_u (local.get $pos) (local.get $end))) - (local.set $byte1 (i32.load8_u (local.get $pos))) - - ;; 1-byte: 0xxxxxxx - (if (i32.eqz (i32.and (local.get $byte1) (i32.const 0x80))) - (then - (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $byte1)) - (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) - (local.set $pos (i32.add (local.get $pos) (i32.const 1))) - (br $continue))) - - ;; 2-byte: 110xxxxx 10xxxxxx - (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xE0)) (i32.const 0xC0)) - (then - (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) - (local.set $codePoint - (i32.or - (i32.shl (i32.and (local.get $byte1) (i32.const 0x1F)) (i32.const 6)) - (i32.and (local.get $byte2) (i32.const 0x3F)))) - (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $codePoint)) - (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) - (local.set $pos (i32.add (local.get $pos) (i32.const 2))) - (br $continue))) + (local.set $b1 (i32.load8_u (local.get $pos))) - ;; 3-byte: 1110xxxx 10xxxxxx 10xxxxxx - (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xF0)) (i32.const 0xE0)) + (if (i32.eqz (i32.and (local.get $b1) (i32.const 0x80))) (then - (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) - (local.set $byte3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) - (local.set $codePoint - (i32.or - (i32.or - (i32.shl (i32.and (local.get $byte1) (i32.const 0x0F)) (i32.const 12)) - (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 6))) - (i32.and (local.get $byte3) (i32.const 0x3F)))) - (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $codePoint)) + ;; 1-byte: 0xxxxxxx + (array.set $i16_array (local.get $arr) (local.get $outIdx) (local.get $b1)) (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) - (local.set $pos (i32.add (local.get $pos) (i32.const 3))) - (br $continue))) - - ;; 4-byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - (if (i32.eq (i32.and (local.get $byte1) (i32.const 0xF8)) (i32.const 0xF0)) - (then - (local.set $byte2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) - (local.set $byte3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) - (local.set $byte4 (i32.load8_u (i32.add (local.get $pos) (i32.const 3)))) - (local.set $codePoint - (i32.or - (i32.or + (local.set $pos (i32.add (local.get $pos) (i32.const 1)))) + (else + (if (i32.eq (i32.and (local.get $b1) (i32.const 0xE0)) (i32.const 0xC0)) + (then + ;; 2-byte: 110xxxxx 10xxxxxx + (local.set $b2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (array.set $i16_array (local.get $arr) (local.get $outIdx) (i32.or - (i32.shl (i32.and (local.get $byte1) (i32.const 0x07)) (i32.const 18)) - (i32.shl (i32.and (local.get $byte2) (i32.const 0x3F)) (i32.const 12))) - (i32.shl (i32.and (local.get $byte3) (i32.const 0x3F)) (i32.const 6))) - (i32.and (local.get $byte4) (i32.const 0x3F)))) - ;; Convert to surrogate pair - (local.set $codePoint (i32.sub (local.get $codePoint) (i32.const 0x10000))) - ;; High surrogate - (array.set $i16_array (local.get $arr) (local.get $outIdx) - (i32.or - (i32.shr_u (local.get $codePoint) (i32.const 10)) - (i32.const 0xD800))) - (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) - ;; Low surrogate - (array.set $i16_array (local.get $arr) (local.get $outIdx) - (i32.or - (i32.and (local.get $codePoint) (i32.const 0x3FF)) - (i32.const 0xDC00))) - (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) - (local.set $pos (i32.add (local.get $pos) (i32.const 4))) - (br $continue))) + (i32.shl (i32.and (local.get $b1) (i32.const 0x1F)) (i32.const 6)) + (i32.and (local.get $b2) (i32.const 0x3F)))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) + (local.set $pos (i32.add (local.get $pos) (i32.const 2)))) + (else + (if (i32.eq (i32.and (local.get $b1) (i32.const 0xF0)) (i32.const 0xE0)) + (then + ;; 3-byte: 1110xxxx 10xxxxxx 10xxxxxx + (local.set $b2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (local.set $b3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) + (array.set $i16_array (local.get $arr) (local.get $outIdx) + (i32.or + (i32.or + (i32.shl (i32.and (local.get $b1) (i32.const 0x0F)) (i32.const 12)) + (i32.shl (i32.and (local.get $b2) (i32.const 0x3F)) (i32.const 6))) + (i32.and (local.get $b3) (i32.const 0x3F)))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) + (local.set $pos (i32.add (local.get $pos) (i32.const 3)))) + (else + (if (i32.eq (i32.and (local.get $b1) (i32.const 0xF8)) (i32.const 0xF0)) + (then + ;; 4-byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + (local.set $b2 (i32.load8_u (i32.add (local.get $pos) (i32.const 1)))) + (local.set $b3 (i32.load8_u (i32.add (local.get $pos) (i32.const 2)))) + (local.set $b4 (i32.load8_u (i32.add (local.get $pos) (i32.const 3)))) + (local.set $cp + (i32.sub + (i32.or + (i32.or + (i32.or + (i32.shl (i32.and (local.get $b1) (i32.const 0x07)) (i32.const 18)) + (i32.shl (i32.and (local.get $b2) (i32.const 0x3F)) (i32.const 12))) + (i32.shl (i32.and (local.get $b3) (i32.const 0x3F)) (i32.const 6))) + (i32.and (local.get $b4) (i32.const 0x3F))) + (i32.const 0x10000))) + ;; High surrogate + (array.set $i16_array (local.get $arr) (local.get $outIdx) + (i32.or (i32.const 0xD800) (i32.shr_u (local.get $cp) (i32.const 10)))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) + ;; Low surrogate + (array.set $i16_array (local.get $arr) (local.get $outIdx) + (i32.or (i32.const 0xDC00) (i32.and (local.get $cp) (i32.const 0x3FF)))) + (local.set $outIdx (i32.add (local.get $outIdx) (i32.const 1))) + (local.set $pos (i32.add (local.get $pos) (i32.const 4)))) + (else + ;; Invalid byte, skip + (local.set $pos (i32.add (local.get $pos) (i32.const 1))))))))))) - ;; Invalid byte, skip - (local.set $pos (i32.add (local.get $pos) (i32.const 1))) (br $continue))) - ;; Return number of code units written (local.get $outIdx)) - ;; Allocate a GC array for UTF-16 code units + ;; Allocate GC array for UTF-16 code units (func (export "allocArray") (param $size i32) (result (ref $i16_array)) (array.new $i16_array (i32.const 0) (local.get $size))) From 99d10a6a861a430cbd4abd3a23ad78c4b7aad0e9 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sun, 28 Dec 2025 13:18:20 +0900 Subject: [PATCH 13/13] update doc --- package.json | 2 +- wasm/README.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 37417809..8a2a37ca 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "sideEffects": false, "scripts": { "build": "npm publish --dry-run", - "prepare": "npm run clean && webpack --bail && tsgo --build tsconfig.dist.cjs.json tsconfig.dist.esm.json && tsimp tools/fix-ext.mts --mjs dist.esm/*.js dist.esm/*/*.js dist.esm/*.d.ts dist.esm/*/*.d.ts && tsimp tools/fix-ext.mts --cjs dist.cjs/*.js dist.cjs/*/*.js dist.cjs/*.d.ts dist.cjs/*/*.d.ts", + "prepare": "npm run clean && ./wasm/build.sh && webpack --bail && tsgo --build tsconfig.dist.cjs.json tsconfig.dist.esm.json && tsimp tools/fix-ext.mts --mjs dist.esm/*.js dist.esm/*/*.js dist.esm/*.d.ts dist.esm/*/*.d.ts && tsimp tools/fix-ext.mts --cjs dist.cjs/*.js dist.cjs/*/*.js dist.cjs/*.d.ts dist.cjs/*/*.d.ts", "prepublishOnly": "npm run test:dist", "clean": "rimraf build dist dist.*", "test": "mocha 'test/**/*.test.ts'", diff --git a/wasm/README.md b/wasm/README.md index 4372876f..b3bc3e24 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -51,6 +51,38 @@ Three-tier dispatch based on string/byte length: | 51-1000 | WASM | Optimal for medium strings | | > 1000 | TextEncoder/TextDecoder | SIMD-optimized for bulk | +## Optimization Attempts (2025) + +Several optimization approaches were tested for `utf8Count`: + +### 1. Bulk Array Copy (intoCharCodeArray) + +**Hypothesis**: Replace N `charCodeAt` calls with 1 bulk `intoCharCodeArray` + N array reads. + +**Result**: 17-29% slower. GC array allocation overhead outweighs boundary-crossing savings. + +### 2. codePointAt Instead of charCodeAt + +**Hypothesis**: Simplify surrogate pair handling with `codePointAt`. + +**Result**: Slightly slower. `codePointAt` does more internal work to decode surrogates. + +### 3. SIMD Processing + +**Hypothesis**: Copy to linear memory, then use SIMD to process 8 chars at once. + +**Result**: 23-49% slower. The O(n) copy from GC array to linear memory negates SIMD gains. + +``` +JS String β†’ GC Array (1 call) β†’ Linear Memory (N scalar ops) β†’ SIMD + ↑ + This kills SIMD +``` + +### Conclusion + +The scalar `charCodeAt` loop is already near-optimal. The `js-string-builtins` implementation is highly optimized, making per-character calls very cheap. The 2-3x speedup over pure JS is about as good as it gets with current WASM capabilities. + ## References - [js-string-builtins proposal](https://github.com/WebAssembly/js-string-builtins)