Add support for binary JPEG rects, as well as some other minor tweaks

CollabVMSharp/CollabVMClient.cs

@@ -14,6 +14,8 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using CollabVMSharp.Protocol;
+using MsgPack.Serialization;
 using Timer = System.Timers.Timer;
 // ReSharper disable FieldCanBeMadeReadOnly.Local
 // ReSharper disable ArrangeObjectCreationWhenTypeNotEvident
@@ -21,7 +23,9 @@ using Timer = System.Timers.Timer;
 // ReSharper disable ArrangeObjectCreationWhenTypeEvident
 
 namespace CollabVMSharp;
-public class CollabVMClient {
+public class CollabVMClient
+{
+    private static readonly string[] SupportedCapabilities = { "bin" };
     // Fields
     private Uri url;
     private string? username;
@@ -43,6 +47,7 @@ public class CollabVMClient {
     private Dictionary<string, Action<string, string>> commandsOneArg;
     private bool _usesAccountAuth;
     private Dictionary<string,string> _headers;
+    private MessagePackSerializer<CollabVMProtocolMessage> _msgpack;
     // Tasks and related
     private TaskCompletionSource<Node[]> GotNodeList;
     private TaskCompletionSource<bool> GotConnectionToNode;
@@ -152,6 +157,7 @@ public class CollabVMClient {
         this.GotIPTasks = new();
         this.QEMUMonitorResult = new();
         this.QEMUMonitorSemaphore = new(1, 1);
+        this._msgpack = MessagePackSerializer.Get<CollabVMProtocolMessage>();
         // Assign empty handlers to prevent exception
         Chat += delegate { };
         ChatHistory += delegate { };
@@ -188,6 +194,8 @@ public class CollabVMClient {
             this.SendMsg(Guacutils.Encode("rename", this.username));
         else
             this.SendMsg(Guacutils.Encode("rename"));
+        if (SupportedCapabilities.Length > 0)
+            this.SendMsg(Guacutils.Encode(SupportedCapabilities.Prepend("cap").ToArray()));
         this.NOPRecieve.Start();
         this.WebSocketLoop();
         if (this.node == null) return;
@@ -199,15 +207,13 @@ public class CollabVMClient {
     private async void WebSocketLoop() {
         ArraySegment<byte> receivebuffer = new ArraySegment<byte>(new byte[8192]);
         do {
-            MemoryStream ms = new();
+            using var ms = new MemoryStream();
             WebSocketReceiveResult res;
             do {
                 try {
                     res = await socket.ReceiveAsync(receivebuffer, CancellationToken.None);
                 } catch (WebSocketException e) {
-#if DEBUG
+                    Error.Invoke(this, $"Got {e.Message} while reading from WebSocket, closing connection");
-                    Console.Error.WriteLine($"Got {e.Message} while reading from WebSocket, closing connection");
-#endif
                     Cleanup(true);
                     return;
                 }
@@ -218,28 +224,72 @@ public class CollabVMClient {
                 }
                 await ms.WriteAsync(receivebuffer.Array, 0, res.Count);
             } while (!res.EndOfMessage);
-            string msg;
+            if (res.MessageType == WebSocketMessageType.Text)
-            try {
+            {
-                msg = Encoding.UTF8.GetString(ms.ToArray());
+                string msg;
-            } catch (Exception e) {
+                try {
-                #if DEBUG
+                    msg = Encoding.UTF8.GetString(ms.ToArray());
-                await Console.Error.WriteLineAsync($"Failed to read message from socket: {e.Message}");
+                } catch (Exception e) {
-                #endif
+                    Error.Invoke(this, $"Failed to read message from socket: {e.Message}");
-                continue;
+                    continue;
-            } finally {ms.Dispose();}
+                }
-            this.ProcessMessage(msg);
+                this.ProcessMessage(msg);
+            } else if (res.MessageType == WebSocketMessageType.Binary)
+            {
+                ms.Position = 0;
+                await this.ProcessBinaryMessage(ms);
+            }
+
         } while (socket.State == WebSocketState.Open);
         this.Cleanup();
     }
 
+    private async Task ProcessBinaryMessage(MemoryStream data)
+    {
+        CollabVMProtocolMessage msg;
+        try
+        {
+            msg = await this._msgpack.UnpackAsync(data);
+        }
+        catch (Exception ex)
+        {
+            Error.Invoke(this, $"Failed to unpack binary message: {ex.Message}");
+            return;
+        }
+
+        switch (msg.type)
+        {
+            case CollabVMProtocolMessageType.rect:
+            {
+                if (msg.rect == null || msg.rect!.data == null) return;
+                Image rect;
+                try
+                {
+                    rect = Image.Load(msg.rect.data);
+                }
+                catch (Exception ex)
+                {
+                    Error.Invoke(this, "Server sent an invalid screen rect");
+                    return;
+                }
+                framebuffer.Mutate(f => f.DrawImage(rect, new Point(msg.rect.x, msg.rect.y), 1));
+                this.Rect.Invoke(this, new RectEventArgs
+                {
+                    X = msg.rect.x,
+                    Y = msg.rect.y,
+                    Data = rect,
+                });
+                break;
+            }
+        }
+    }
+
     private async void ProcessMessage(string msg) {
         string[] msgArr;
         try {
             msgArr = Guacutils.Decode(msg);
         } catch (Exception e) {
-            #if DEBUG
+            Error.Invoke(this, $"Failed to decode incoming message: {e.Message}");
-            await Console.Error.WriteLineAsync($"Failed to decode incoming message: {e.Message}");
-            #endif
             return;
         }
 
@@ -327,7 +377,7 @@ public class CollabVMClient {
                 {
                     rect = Image.Load(Convert.FromBase64String(msgArr[5]));
                 }
-                catch (FormatException ex)
+                catch (Exception ex)
                 {
                     Error.Invoke(this, "Server sent an invalid screen rect");
                     return;

CollabVMSharp/CollabVMSharp.csproj

@@ -3,7 +3,7 @@
     <PropertyGroup>
         <TargetFramework>net6.0</TargetFramework>
         <LangVersion>10</LangVersion>
-        <Version>2.7.0</Version>
+        <Version>2.7.1</Version>
         <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
         <PackageLicenseExpression>GPL-3.0-or-later</PackageLicenseExpression>
         <PackageReadmeFile>README.md</PackageReadmeFile>
@@ -11,6 +11,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="MsgPack.Cli" Version="1.0.1" />
       <PackageReference Include="SixLabors.ImageSharp" Version="3.1.3" />
     </ItemGroup>
     

CollabVMSharp/Protocol/CollabVMProtocolMessage.cs

@@ -0,0 +1,7 @@
+namespace CollabVMSharp.Protocol;
+
+public class CollabVMProtocolMessage
+{
+    public CollabVMProtocolMessageType type { get; set; }
+    public CollabVMRectMessage? rect { get; set; }
+}

CollabVMSharp/Protocol/CollabVMProtocolMessageType.cs

@@ -0,0 +1,6 @@
+namespace CollabVMSharp.Protocol;
+
+public enum CollabVMProtocolMessageType
+{
+    rect = 0
+}

CollabVMSharp/Protocol/CollabVMRectMessage.cs

@@ -0,0 +1,8 @@
+namespace CollabVMSharp.Protocol;
+
+public class CollabVMRectMessage
+{
+    public int x { get; set; }
+    public int y { get; set; }
+    public byte[] data { get; set; }
+}