diff --git a/.gitignore b/.gitignore index 24eb63b..423961c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ node_modules/ **/dist/ **/.parcel-cache/ -server/config.toml \ No newline at end of file +server/config.toml + +msagent.js/obj diff --git a/msagent.js/Makefile b/msagent.js/Makefile new file mode 100644 index 0000000..0c78091 --- /dev/null +++ b/msagent.js/Makefile @@ -0,0 +1,34 @@ +# Makefile for WASM decompression. + +CXX = clang++ --target=wasm32 +CXXFLAGS = -Wall \ + -Os \ + -nostdlib \ + -fvisibility=hidden \ + -std=c++20 \ + -ffunction-sections \ + -fdata-sections + +src/decompress.wasm: obj/ obj/decompress.o + wasm-ld \ + -o $@ \ + --no-entry \ + --strip-all \ + --export-dynamic \ + --allow-undefined \ + --initial-memory=131072 \ + --error-limit=0 \ + --lto-O3 \ + -O3 \ + --gc-sections \ + obj/decompress.o + + +obj/%.o: src/%.cpp + $(CXX) -c $(CXXFLAGS) $< -o $@ + +clean: + rm -rf obj src/decompress.wasm + +obj/: + mkdir -p obj/ diff --git a/msagent.js/src/decompress.cpp b/msagent.js/src/decompress.cpp new file mode 100644 index 0000000..e6aefca --- /dev/null +++ b/msagent.js/src/decompress.cpp @@ -0,0 +1,118 @@ + +// Integer types from WASI-libc +using usize = unsigned int; +using u32 = unsigned int; +using u8 = unsigned char; + +#define LOWORD(x) (x & 0xffff) +#define LOBYTE(x) (x & 0xff) + +#define PUBLIC __attribute__((visibility("default"))) extern "C" + +PUBLIC usize agentDecompressWASM(const void* pSrcData, usize pSrcSize, void* pTrgData, usize pTrgSize) { + const u8* lSrcPtr = (const u8*)pSrcData; + const u8* lSrcEnd = lSrcPtr + pSrcSize; + u8* lTrgPtr = (u8*)pTrgData; + u8* lTrgEnd = lTrgPtr + pTrgSize; + u32 lSrcQuad; + u8 lTrgByte; + u32 lBitCount = 0; + u32 lSrcOffset; + u32 lRunLgth; + u32 lRunCount; + + if((pSrcSize <= 7) || (*lSrcPtr != 0)) { + return 0; + } + + for(lBitCount = 1; (*(lSrcEnd - lBitCount) == 0xFF); lBitCount++) { + if(lBitCount > 6) { + break; + } + } + if(lBitCount < 6) { + return 0; + } + + lBitCount = 0; + lSrcPtr += 5; + + while((lSrcPtr < lSrcEnd) && (lTrgPtr < lTrgEnd)) { + lSrcQuad = *(const u32*)(lSrcPtr - sizeof(u32)); + + if(lSrcQuad & (1 << LOWORD(lBitCount))) { + lSrcOffset = 1; + + if(lSrcQuad & (1 << LOWORD(lBitCount + 1))) { + if(lSrcQuad & (1 << LOWORD(lBitCount + 2))) { + if(lSrcQuad & (1 << LOWORD(lBitCount + 3))) { + lSrcQuad >>= LOWORD(lBitCount + 4); + lSrcQuad &= 0x000FFFFF; + if(lSrcQuad == 0x000FFFFF) { + break; + } + lSrcQuad += 4673; + lBitCount += 24; + + lSrcOffset = 2; + } else { + lSrcQuad >>= LOWORD(lBitCount + 4); + lSrcQuad &= 0x00000FFF; + lSrcQuad += 577; + lBitCount += 16; + } + } else { + lSrcQuad >>= LOWORD(lBitCount + 3); + lSrcQuad &= 0x000001FF; + lSrcQuad += 65; + lBitCount += 12; + } + } else { + lSrcQuad >>= LOWORD(lBitCount + 2); + lSrcQuad &= 0x0000003F; + lSrcQuad += 1; + lBitCount += 8; + } + + lSrcPtr += (lBitCount / 8); + lBitCount &= 7; + lRunLgth = *(const u32*)(lSrcPtr - sizeof(u32)); + lRunCount = 0; + while(lRunLgth & (1 << LOWORD(lBitCount + lRunCount))) { + lRunCount++; + if(lRunCount > 11) { + break; + } + } + + lRunLgth >>= LOWORD(lBitCount + lRunCount + 1); + lRunLgth &= (1 << lRunCount) - 1; + lRunLgth += 1 << lRunCount; + lRunLgth += lSrcOffset; + lBitCount += lRunCount * 2 + 1; + + if(lTrgPtr + lRunLgth > lTrgEnd) { + break; + } + if(lTrgPtr - lSrcQuad < pTrgData) { + break; + } + while((long)lRunLgth > 0) { + lTrgByte = *(lTrgPtr - lSrcQuad); + *(lTrgPtr++) = lTrgByte; + lRunLgth--; + } + } else { + lSrcQuad >>= LOWORD(lBitCount + 1); + lBitCount += 9; + + lTrgByte = LOBYTE(lSrcQuad); + *(lTrgPtr++) = lTrgByte; + } + + lSrcPtr += lBitCount / 8; + lBitCount &= 7; + } + + return (usize)(lTrgPtr - (u8*)pTrgData); +} diff --git a/msagent.js/src/decompress.ts b/msagent.js/src/decompress.ts index 1e51fd3..a4bbd03 100644 --- a/msagent.js/src/decompress.ts +++ b/msagent.js/src/decompress.ts @@ -1,110 +1,58 @@ // Please note that the "meaningless" shifts of 0 are to force // the value to be a 32-bit integer. Do not remove them. -function LOWORD(n: number) { - return (n >>> 0) & 0xffff; +let compressWasm: WebAssembly.WebAssemblyInstantiatedSource; + +interface CompressWasmExports { + memory: WebAssembly.Memory; + agentDecompressWASM: any; } -function LOBYTE(n: number) { - return (n >>> 0) & 0xff; +// Initalize the decompression module +export async function compressInit() { + let url = new URL('decompress.wasm', import.meta.url); + compressWasm = await WebAssembly.instantiateStreaming(fetch(url)); } -function HIWORD(n: number) { - return (n >>> 16) & 0xffff; + +function compressWasmGetExports() { + return (compressWasm.instance.exports as any) as CompressWasmExports; } +function compressWASMGetMemory() : WebAssembly.Memory { + return compressWasmGetExports().memory; +} + + +// debugging +//(window as any).DEBUGcompressGetWASM = () => { +// return compressWasm; +//} + // Decompress Agent compressed data. This compression algorithm sucks. // [dest] is to be preallocated to the decompressed data size. export function compressDecompress(src: Uint8Array, dest: Uint8Array) { - let bitCount = 0; - let srcPtr = 0; - let destPtr = 0; - let srcOffset = 0; - - let dv = new DataView(src.buffer, src.byteOffset, src.byteLength); - - let putb = (b: number) => (dest[destPtr++] = b); - - // Make sure the bitstream is valid - if (src.length <= 7 || src[0] != 0) return 0; - - for (bitCount = 1; src[src.length - bitCount] == 0xff; bitCount++) { - if (bitCount > 6) break; + // Grow the WASM heap if needed. Funnily enough, this code is never hit in most + // ACSes, so IDK if it's even needed + let memory = compressWASMGetMemory(); + if(memory.buffer.byteLength < src.length + dest.length) { + // A WebAssembly page is 64kb, so we need to grow at least that much + let npages = Math.floor((src.length + dest.length) / 65535) + 1; + console.log("Need to grow WASM heap", npages, "pages", "(current byteLength is", memory.buffer.byteLength, ", we need", src.length + dest.length, ")"); + memory.grow(npages); } - if (bitCount < 6) return 0; + let copyBuffer = new Uint8Array(memory.buffer); - bitCount = 0; - srcPtr += 5; + // Copy source to memory[0]. This will make things a bit simpler + copyBuffer.set(src, 0); - while (srcPtr < src.length && destPtr < dest.length) { - let quad = dv.getUint32(srcPtr - 4, true); + // Call the WASM compression routine + let nrBytesDecompressed = compressWasmGetExports().agentDecompressWASM(0, src.length, src.length, dest.length); - if (quad & (1 << LOWORD(bitCount))) { - srcOffset = 1; + if(nrBytesDecompressed != dest.length) + throw new Error(`decompression failed: ${nrBytesDecompressed} != ${dest.length}`); - if (quad & (1 << LOWORD(bitCount + 1))) { - if (quad & (1 << LOWORD(bitCount + 2))) { - if (quad & (1 << LOWORD(bitCount + 3))) { - quad >>= LOWORD(bitCount + 4); - quad &= 0x000fffff; - - // End of compressed bitstream - if (quad == 0x000fffff) break; - - quad += 4673; - bitCount += 24; - srcOffset = 2; - } else { - quad >>= LOWORD(bitCount + 4); - quad &= 0x0000fff; - quad += 577; - bitCount += 16; - } - } else { - quad >>= LOWORD(bitCount + 3); - quad &= 0x000001ff; - quad += 65; - bitCount += 12; - } - } else { - quad >>= LOWORD(bitCount + 2); - quad &= 0x0000003f; - quad += 1; - bitCount += 8; - } - - srcPtr += bitCount / 8; - bitCount &= 7; - let runCount = 0; - let runLength = dv.getUint32(srcPtr - 4, true); - - while (runLength & (1 << LOWORD(bitCount + runCount))) { - runCount++; - - if (runCount > 11) break; - } - - runLength >>= LOWORD(bitCount + runCount + 1); - runLength &= (1 << runCount) - 1; - runLength += 1 << runCount; - runLength += srcOffset; - bitCount = runCount * 2 + 1; - - if (destPtr + runLength > dest.length) break; - - while (runLength > 0) { - putb(dest[destPtr - quad]); - runLength--; - } - } else { - // a literal byte - quad >>= LOWORD(bitCount + 1); - bitCount += 9; - putb(LOBYTE(quad)); - } - - srcPtr += bitCount / 8; - bitCount &= 7; - } + // Dest will be memory[src.length..dest.length] + dest.set(copyBuffer.slice(src.length, dest.length), 0); } diff --git a/msagent.js/src/decompress.wasm b/msagent.js/src/decompress.wasm new file mode 100755 index 0000000..5cec002 Binary files /dev/null and b/msagent.js/src/decompress.wasm differ diff --git a/msagent.js/src/index.ts b/msagent.js/src/index.ts index a1cba3b..3d3c4e0 100644 --- a/msagent.js/src/index.ts +++ b/msagent.js/src/index.ts @@ -1,3 +1,4 @@ +import { compressInit } from "./decompress.js"; import { wordballoonInit } from "./wordballoon.js"; export * from "./types.js"; @@ -9,5 +10,6 @@ export * from "./wordballoon.js"; // Convinence function which initalizes all of msagent.js. export async function agentInit() { + await compressInit(); await wordballoonInit(); }