msagent.js: Rewrite decompression to use WASM

Mostly so it actually works. I couldn't get a fully JS version working and I can't be bothered to when this seems to work.

Note that yes, the wasm IS checked in to the repository. This is just so clang isn't a direct build dependency, but will be needed if the decompression module needs to be updated.

.gitignore

@@ -15,4 +15,6 @@
 node_modules/
 **/dist/
 **/.parcel-cache/
-server/config.toml
+server/config.toml
+
+msagent.js/obj

msagent.js/Makefile

@@ -0,0 +1,34 @@
+# Makefile for WASM decompression.
+
+CXX = clang++ --target=wasm32
+CXXFLAGS = -Wall \
+		-Os \
+		-nostdlib \
+		-fvisibility=hidden \
+		-std=c++20 \
+		-ffunction-sections \
+		-fdata-sections
+
+src/decompress.wasm: obj/ obj/decompress.o
+	wasm-ld \
+		-o $@ \
+		--no-entry \
+		--strip-all \
+		--export-dynamic \
+		--allow-undefined \
+		--initial-memory=131072 \
+		--error-limit=0 \
+		--lto-O3 \
+		-O3 \
+		--gc-sections \
+		obj/decompress.o
+
+
+obj/%.o: src/%.cpp
+	$(CXX) -c $(CXXFLAGS) $< -o $@
+
+clean:
+	rm -rf obj src/decompress.wasm
+
+obj/:
+	mkdir -p obj/

msagent.js/src/decompress.cpp

@@ -0,0 +1,118 @@
+
+// Integer types from WASI-libc
+using usize = unsigned int;
+using u32 = unsigned int;
+using u8 = unsigned char;
+
+#define LOWORD(x) (x & 0xffff)
+#define LOBYTE(x) (x & 0xff)
+
+#define PUBLIC __attribute__((visibility("default"))) extern "C"
+
+PUBLIC usize agentDecompressWASM(const void* pSrcData, usize pSrcSize, void* pTrgData, usize pTrgSize) {
+	const u8* lSrcPtr = (const u8*)pSrcData;
+	const u8* lSrcEnd = lSrcPtr + pSrcSize;
+	u8* lTrgPtr = (u8*)pTrgData;
+	u8* lTrgEnd = lTrgPtr + pTrgSize;
+	u32 lSrcQuad;
+	u8 lTrgByte;
+	u32 lBitCount = 0;
+	u32 lSrcOffset;
+	u32 lRunLgth;
+	u32 lRunCount;
+
+	if((pSrcSize <= 7) || (*lSrcPtr != 0)) {
+		return 0;
+	}
+
+	for(lBitCount = 1; (*(lSrcEnd - lBitCount) == 0xFF); lBitCount++) {
+		if(lBitCount > 6) {
+			break;
+		}
+	}
+	if(lBitCount < 6) {
+		return 0;
+	}
+
+	lBitCount = 0;
+	lSrcPtr += 5;
+
+	while((lSrcPtr < lSrcEnd) && (lTrgPtr < lTrgEnd)) {
+		lSrcQuad = *(const u32*)(lSrcPtr - sizeof(u32));
+
+		if(lSrcQuad & (1 << LOWORD(lBitCount))) {
+			lSrcOffset = 1;
+
+			if(lSrcQuad & (1 << LOWORD(lBitCount + 1))) {
+				if(lSrcQuad & (1 << LOWORD(lBitCount + 2))) {
+					if(lSrcQuad & (1 << LOWORD(lBitCount + 3))) {
+						lSrcQuad >>= LOWORD(lBitCount + 4);
+						lSrcQuad &= 0x000FFFFF;
+						if(lSrcQuad == 0x000FFFFF) {
+							break;
+						}
+						lSrcQuad += 4673;
+						lBitCount += 24;
+
+						lSrcOffset = 2;
+					} else {
+						lSrcQuad >>= LOWORD(lBitCount + 4);
+						lSrcQuad &= 0x00000FFF;
+						lSrcQuad += 577;
+						lBitCount += 16;
+					}
+				} else {
+					lSrcQuad >>= LOWORD(lBitCount + 3);
+					lSrcQuad &= 0x000001FF;
+					lSrcQuad += 65;
+					lBitCount += 12;
+				}
+			} else {
+				lSrcQuad >>= LOWORD(lBitCount + 2);
+				lSrcQuad &= 0x0000003F;
+				lSrcQuad += 1;
+				lBitCount += 8;
+			}
+
+			lSrcPtr += (lBitCount / 8);
+			lBitCount &= 7;
+			lRunLgth = *(const u32*)(lSrcPtr - sizeof(u32));
+			lRunCount = 0;
+			while(lRunLgth & (1 << LOWORD(lBitCount + lRunCount))) {
+				lRunCount++;
+				if(lRunCount > 11) {
+					break;
+				}
+			}
+
+			lRunLgth >>= LOWORD(lBitCount + lRunCount + 1);
+			lRunLgth &= (1 << lRunCount) - 1;
+			lRunLgth += 1 << lRunCount;
+			lRunLgth += lSrcOffset;
+			lBitCount += lRunCount * 2 + 1;
+
+			if(lTrgPtr + lRunLgth > lTrgEnd) {
+				break;
+			}
+			if(lTrgPtr - lSrcQuad < pTrgData) {
+				break;
+			}
+			while((long)lRunLgth > 0) {
+				lTrgByte = *(lTrgPtr - lSrcQuad);
+				*(lTrgPtr++) = lTrgByte;
+				lRunLgth--;
+			}
+		} else {
+			lSrcQuad >>= LOWORD(lBitCount + 1);
+			lBitCount += 9;
+
+			lTrgByte = LOBYTE(lSrcQuad);
+			*(lTrgPtr++) = lTrgByte;
+		}
+
+		lSrcPtr += lBitCount / 8;
+		lBitCount &= 7;
+	}
+
+	return (usize)(lTrgPtr - (u8*)pTrgData);
+}

msagent.js/src/decompress.ts

@@ -1,110 +1,58 @@
 // Please note that the "meaningless" shifts of 0 are to force
 // the value to be a 32-bit integer. Do not remove them.
 
-function LOWORD(n: number) {
+let compressWasm: WebAssembly.WebAssemblyInstantiatedSource;
-	return (n >>> 0) & 0xffff;
+
+interface CompressWasmExports {
+	memory: WebAssembly.Memory;
+	agentDecompressWASM: any;
 }
 
-function LOBYTE(n: number) {
+// Initalize the decompression module
-	return (n >>> 0) & 0xff;
+export async function compressInit() {
+	let url = new URL('decompress.wasm', import.meta.url);
+	compressWasm = await WebAssembly.instantiateStreaming(fetch(url));
 }
 
-function HIWORD(n: number) {
+
-	return (n >>> 16) & 0xffff;
+function compressWasmGetExports() {
+	return (compressWasm.instance.exports as any) as CompressWasmExports;
 }
 
+function compressWASMGetMemory() : WebAssembly.Memory {
+	return compressWasmGetExports().memory;
+}
+
+
+// debugging
+//(window as any).DEBUGcompressGetWASM = () => {
+//	return compressWasm;
+//}
+
 // Decompress Agent compressed data. This compression algorithm sucks.
 // [dest] is to be preallocated to the decompressed data size.
 export function compressDecompress(src: Uint8Array, dest: Uint8Array) {
-	let bitCount = 0;
+	// Grow the WASM heap if needed. Funnily enough, this code is never hit in most
-	let srcPtr = 0;
+	// ACSes, so IDK if it's even needed
-	let destPtr = 0;
+	let memory = compressWASMGetMemory();
-	let srcOffset = 0;
+	if(memory.buffer.byteLength < src.length + dest.length) {
-
+		// A WebAssembly page is 64kb, so we need to grow at least that much
-	let dv = new DataView(src.buffer, src.byteOffset, src.byteLength);
+		let npages = Math.floor((src.length + dest.length) / 65535) + 1;
-
+		console.log("Need to grow WASM heap", npages, "pages", "(current byteLength is", memory.buffer.byteLength, ", we need", src.length + dest.length, ")");
-	let putb = (b: number) => (dest[destPtr++] = b);
+		memory.grow(npages);
-
-	// Make sure the bitstream is valid
-	if (src.length <= 7 || src[0] != 0) return 0;
-
-	for (bitCount = 1; src[src.length - bitCount] == 0xff; bitCount++) {
-		if (bitCount > 6) break;
 	}
 
-	if (bitCount < 6) return 0;
+	let copyBuffer = new Uint8Array(memory.buffer);
 
-	bitCount = 0;
+	// Copy source to memory[0]. This will make things a bit simpler
-	srcPtr += 5;
+	copyBuffer.set(src, 0);
 
-	while (srcPtr < src.length && destPtr < dest.length) {
+	// Call the WASM compression routine
-		let quad = dv.getUint32(srcPtr - 4, true);
+	let nrBytesDecompressed = compressWasmGetExports().agentDecompressWASM(0, src.length, src.length, dest.length);
 
-		if (quad & (1 << LOWORD(bitCount))) {
+	if(nrBytesDecompressed != dest.length)
-			srcOffset = 1;
+		throw new Error(`decompression failed: ${nrBytesDecompressed} != ${dest.length}`);
 
-			if (quad & (1 << LOWORD(bitCount + 1))) {
+	// Dest will be memory[src.length..dest.length]
-				if (quad & (1 << LOWORD(bitCount + 2))) {
+	dest.set(copyBuffer.slice(src.length, dest.length), 0);
-					if (quad & (1 << LOWORD(bitCount + 3))) {
-						quad >>= LOWORD(bitCount + 4);
-						quad &= 0x000fffff;
-
-						// End of compressed bitstream
-						if (quad == 0x000fffff) break;
-
-						quad += 4673;
-						bitCount += 24;
-						srcOffset = 2;
-					} else {
-						quad >>= LOWORD(bitCount + 4);
-						quad &= 0x0000fff;
-						quad += 577;
-						bitCount += 16;
-					}
-				} else {
-					quad >>= LOWORD(bitCount + 3);
-					quad &= 0x000001ff;
-					quad += 65;
-					bitCount += 12;
-				}
-			} else {
-				quad >>= LOWORD(bitCount + 2);
-				quad &= 0x0000003f;
-				quad += 1;
-				bitCount += 8;
-			}
-
-			srcPtr += bitCount / 8;
-			bitCount &= 7;
-			let runCount = 0;
-			let runLength = dv.getUint32(srcPtr - 4, true);
-
-			while (runLength & (1 << LOWORD(bitCount + runCount))) {
-				runCount++;
-
-				if (runCount > 11) break;
-			}
-
-			runLength >>= LOWORD(bitCount + runCount + 1);
-			runLength &= (1 << runCount) - 1;
-			runLength += 1 << runCount;
-			runLength += srcOffset;
-			bitCount = runCount * 2 + 1;
-
-			if (destPtr + runLength > dest.length) break;
-
-			while (runLength > 0) {
-				putb(dest[destPtr - quad]);
-				runLength--;
-			}
-		} else {
-			// a literal byte
-			quad >>= LOWORD(bitCount + 1);
-			bitCount += 9;
-			putb(LOBYTE(quad));
-		}
-
-		srcPtr += bitCount / 8;
-		bitCount &= 7;
-	}
 }

msagent.js/src/index.ts

@@ -1,3 +1,4 @@
+import { compressInit } from "./decompress.js";
 import { wordballoonInit } from "./wordballoon.js";
 
 export * from "./types.js";
@@ -9,5 +10,6 @@ export * from "./wordballoon.js";
 
 // Convinence function which initalizes all of msagent.js.
 export async function agentInit() {
+    await compressInit();
     await wordballoonInit();
 }