msagent.js: Rewrite decompression to use WASM

Mostly so it actually works. I couldn't get a fully JS version working and I can't be bothered to when this seems to work.

Note that yes, the wasm IS checked in to the repository. This is just so clang isn't a direct build dependency, but will be needed if the decompression module needs to be updated.
This commit is contained in:
Lily Tsuru 2024-07-09 18:53:18 -04:00
parent 9eadf40d4c
commit 5c3433461d
6 changed files with 196 additions and 92 deletions

4
.gitignore vendored
View file

@ -15,4 +15,6 @@
node_modules/
**/dist/
**/.parcel-cache/
server/config.toml
server/config.toml
msagent.js/obj

34
msagent.js/Makefile Normal file
View file

@ -0,0 +1,34 @@
# Makefile for WASM decompression.
CXX = clang++ --target=wasm32
CXXFLAGS = -Wall \
-Os \
-nostdlib \
-fvisibility=hidden \
-std=c++20 \
-ffunction-sections \
-fdata-sections
src/decompress.wasm: obj/ obj/decompress.o
wasm-ld \
-o $@ \
--no-entry \
--strip-all \
--export-dynamic \
--allow-undefined \
--initial-memory=131072 \
--error-limit=0 \
--lto-O3 \
-O3 \
--gc-sections \
obj/decompress.o
obj/%.o: src/%.cpp
$(CXX) -c $(CXXFLAGS) $< -o $@
clean:
rm -rf obj src/decompress.wasm
obj/:
mkdir -p obj/

View file

@ -0,0 +1,118 @@
// Integer types from WASI-libc
using usize = unsigned int;
using u32 = unsigned int;
using u8 = unsigned char;
#define LOWORD(x) (x & 0xffff)
#define LOBYTE(x) (x & 0xff)
#define PUBLIC __attribute__((visibility("default"))) extern "C"
PUBLIC usize agentDecompressWASM(const void* pSrcData, usize pSrcSize, void* pTrgData, usize pTrgSize) {
const u8* lSrcPtr = (const u8*)pSrcData;
const u8* lSrcEnd = lSrcPtr + pSrcSize;
u8* lTrgPtr = (u8*)pTrgData;
u8* lTrgEnd = lTrgPtr + pTrgSize;
u32 lSrcQuad;
u8 lTrgByte;
u32 lBitCount = 0;
u32 lSrcOffset;
u32 lRunLgth;
u32 lRunCount;
if((pSrcSize <= 7) || (*lSrcPtr != 0)) {
return 0;
}
for(lBitCount = 1; (*(lSrcEnd - lBitCount) == 0xFF); lBitCount++) {
if(lBitCount > 6) {
break;
}
}
if(lBitCount < 6) {
return 0;
}
lBitCount = 0;
lSrcPtr += 5;
while((lSrcPtr < lSrcEnd) && (lTrgPtr < lTrgEnd)) {
lSrcQuad = *(const u32*)(lSrcPtr - sizeof(u32));
if(lSrcQuad & (1 << LOWORD(lBitCount))) {
lSrcOffset = 1;
if(lSrcQuad & (1 << LOWORD(lBitCount + 1))) {
if(lSrcQuad & (1 << LOWORD(lBitCount + 2))) {
if(lSrcQuad & (1 << LOWORD(lBitCount + 3))) {
lSrcQuad >>= LOWORD(lBitCount + 4);
lSrcQuad &= 0x000FFFFF;
if(lSrcQuad == 0x000FFFFF) {
break;
}
lSrcQuad += 4673;
lBitCount += 24;
lSrcOffset = 2;
} else {
lSrcQuad >>= LOWORD(lBitCount + 4);
lSrcQuad &= 0x00000FFF;
lSrcQuad += 577;
lBitCount += 16;
}
} else {
lSrcQuad >>= LOWORD(lBitCount + 3);
lSrcQuad &= 0x000001FF;
lSrcQuad += 65;
lBitCount += 12;
}
} else {
lSrcQuad >>= LOWORD(lBitCount + 2);
lSrcQuad &= 0x0000003F;
lSrcQuad += 1;
lBitCount += 8;
}
lSrcPtr += (lBitCount / 8);
lBitCount &= 7;
lRunLgth = *(const u32*)(lSrcPtr - sizeof(u32));
lRunCount = 0;
while(lRunLgth & (1 << LOWORD(lBitCount + lRunCount))) {
lRunCount++;
if(lRunCount > 11) {
break;
}
}
lRunLgth >>= LOWORD(lBitCount + lRunCount + 1);
lRunLgth &= (1 << lRunCount) - 1;
lRunLgth += 1 << lRunCount;
lRunLgth += lSrcOffset;
lBitCount += lRunCount * 2 + 1;
if(lTrgPtr + lRunLgth > lTrgEnd) {
break;
}
if(lTrgPtr - lSrcQuad < pTrgData) {
break;
}
while((long)lRunLgth > 0) {
lTrgByte = *(lTrgPtr - lSrcQuad);
*(lTrgPtr++) = lTrgByte;
lRunLgth--;
}
} else {
lSrcQuad >>= LOWORD(lBitCount + 1);
lBitCount += 9;
lTrgByte = LOBYTE(lSrcQuad);
*(lTrgPtr++) = lTrgByte;
}
lSrcPtr += lBitCount / 8;
lBitCount &= 7;
}
return (usize)(lTrgPtr - (u8*)pTrgData);
}

View file

@ -1,110 +1,58 @@
// Please note that the "meaningless" shifts of 0 are to force
// the value to be a 32-bit integer. Do not remove them.
function LOWORD(n: number) {
return (n >>> 0) & 0xffff;
let compressWasm: WebAssembly.WebAssemblyInstantiatedSource;
interface CompressWasmExports {
memory: WebAssembly.Memory;
agentDecompressWASM: any;
}
function LOBYTE(n: number) {
return (n >>> 0) & 0xff;
// Initalize the decompression module
export async function compressInit() {
let url = new URL('decompress.wasm', import.meta.url);
compressWasm = await WebAssembly.instantiateStreaming(fetch(url));
}
function HIWORD(n: number) {
return (n >>> 16) & 0xffff;
function compressWasmGetExports() {
return (compressWasm.instance.exports as any) as CompressWasmExports;
}
function compressWASMGetMemory() : WebAssembly.Memory {
return compressWasmGetExports().memory;
}
// debugging
//(window as any).DEBUGcompressGetWASM = () => {
// return compressWasm;
//}
// Decompress Agent compressed data. This compression algorithm sucks.
// [dest] is to be preallocated to the decompressed data size.
export function compressDecompress(src: Uint8Array, dest: Uint8Array) {
let bitCount = 0;
let srcPtr = 0;
let destPtr = 0;
let srcOffset = 0;
let dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
let putb = (b: number) => (dest[destPtr++] = b);
// Make sure the bitstream is valid
if (src.length <= 7 || src[0] != 0) return 0;
for (bitCount = 1; src[src.length - bitCount] == 0xff; bitCount++) {
if (bitCount > 6) break;
// Grow the WASM heap if needed. Funnily enough, this code is never hit in most
// ACSes, so IDK if it's even needed
let memory = compressWASMGetMemory();
if(memory.buffer.byteLength < src.length + dest.length) {
// A WebAssembly page is 64kb, so we need to grow at least that much
let npages = Math.floor((src.length + dest.length) / 65535) + 1;
console.log("Need to grow WASM heap", npages, "pages", "(current byteLength is", memory.buffer.byteLength, ", we need", src.length + dest.length, ")");
memory.grow(npages);
}
if (bitCount < 6) return 0;
let copyBuffer = new Uint8Array(memory.buffer);
bitCount = 0;
srcPtr += 5;
// Copy source to memory[0]. This will make things a bit simpler
copyBuffer.set(src, 0);
while (srcPtr < src.length && destPtr < dest.length) {
let quad = dv.getUint32(srcPtr - 4, true);
// Call the WASM compression routine
let nrBytesDecompressed = compressWasmGetExports().agentDecompressWASM(0, src.length, src.length, dest.length);
if (quad & (1 << LOWORD(bitCount))) {
srcOffset = 1;
if(nrBytesDecompressed != dest.length)
throw new Error(`decompression failed: ${nrBytesDecompressed} != ${dest.length}`);
if (quad & (1 << LOWORD(bitCount + 1))) {
if (quad & (1 << LOWORD(bitCount + 2))) {
if (quad & (1 << LOWORD(bitCount + 3))) {
quad >>= LOWORD(bitCount + 4);
quad &= 0x000fffff;
// End of compressed bitstream
if (quad == 0x000fffff) break;
quad += 4673;
bitCount += 24;
srcOffset = 2;
} else {
quad >>= LOWORD(bitCount + 4);
quad &= 0x0000fff;
quad += 577;
bitCount += 16;
}
} else {
quad >>= LOWORD(bitCount + 3);
quad &= 0x000001ff;
quad += 65;
bitCount += 12;
}
} else {
quad >>= LOWORD(bitCount + 2);
quad &= 0x0000003f;
quad += 1;
bitCount += 8;
}
srcPtr += bitCount / 8;
bitCount &= 7;
let runCount = 0;
let runLength = dv.getUint32(srcPtr - 4, true);
while (runLength & (1 << LOWORD(bitCount + runCount))) {
runCount++;
if (runCount > 11) break;
}
runLength >>= LOWORD(bitCount + runCount + 1);
runLength &= (1 << runCount) - 1;
runLength += 1 << runCount;
runLength += srcOffset;
bitCount = runCount * 2 + 1;
if (destPtr + runLength > dest.length) break;
while (runLength > 0) {
putb(dest[destPtr - quad]);
runLength--;
}
} else {
// a literal byte
quad >>= LOWORD(bitCount + 1);
bitCount += 9;
putb(LOBYTE(quad));
}
srcPtr += bitCount / 8;
bitCount &= 7;
}
// Dest will be memory[src.length..dest.length]
dest.set(copyBuffer.slice(src.length, dest.length), 0);
}

BIN
msagent.js/src/decompress.wasm Executable file

Binary file not shown.

View file

@ -1,3 +1,4 @@
import { compressInit } from "./decompress.js";
import { wordballoonInit } from "./wordballoon.js";
export * from "./types.js";
@ -9,5 +10,6 @@ export * from "./wordballoon.js";
// Convinence function which initalizes all of msagent.js.
export async function agentInit() {
await compressInit();
await wordballoonInit();
}