add support for binary JPEG (webapp)

package.json

@@ -15,6 +15,7 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@popperjs/core": "^2.11.8",
+    "@ygoe/msgpack": "^1.0.3",
     "bootstrap": "^5.3.2",
     "dayjs": "^1.11.10",
     "dompurify": "^3.1.0",

src/ts/protocol/CollabVMClient.ts

@@ -9,6 +9,8 @@ import GetKeysym from '../keyboard.js';
 import VoteStatus from './VoteStatus.js';
 import MuteState from './MuteState.js';
 import { StringLike } from '../StringLike.js';
+import msgpack from '@ygoe/msgpack';
+import { CollabVMProtocolMessage, CollabVMProtocolMessageType } from './binaryprotocol/CollabVMProtocolMessage.js';
 const w = window as any;
 
 export interface CollabVMClientEvents {
@@ -51,6 +53,8 @@ interface CollabVMClientPrivateEvents {
 	qemu: (qemuResponse: string) => void;
 }
 
+const DefaultCapabilities = [ "bin" ];
+
 export default class CollabVMClient {
 	// Fields
 	private socket: WebSocket;
@@ -185,6 +189,7 @@ export default class CollabVMClient {
 		this.canvas.addEventListener('contextmenu', (e) => e.preventDefault());
 		// Create the WebSocket
 		this.socket = new WebSocket(url, 'guacamole');
+		this.socket.binaryType = 'arraybuffer';
 		// Add the event listeners
 		this.socket.addEventListener('open', () => this.onOpen());
 		this.socket.addEventListener('message', (event) => this.onMessage(event));
@@ -196,8 +201,37 @@ export default class CollabVMClient {
 		this.internalEmitter.emit('open');
 	}
 
+	private onBinaryMessage(data: ArrayBuffer) {
+		let msg: CollabVMProtocolMessage;
+		try {
+			msg = msgpack.decode(data);
+		} catch {
+			console.error("Server sent invalid binary message");
+			return;
+		}
+		if (msg.type === undefined) return;
+		switch (msg.type) {
+			case CollabVMProtocolMessageType.rect: {
+				if (!msg.rect || msg.rect.x === undefined || msg.rect.y === undefined || msg.rect.data === undefined) return;
+				let blob = new Blob( [ new Uint8Array(msg.rect.data) ], {type: "image/jpeg"});
+				let url = URL.createObjectURL(blob);
+				let img = new Image();
+				img.addEventListener('load', () => {
+					this.loadRectangle(img, msg.rect!.x, msg.rect!.y);
+					URL.revokeObjectURL(url);
+				});
+				img.src = url;
+				break;
+			}
+		}
+	}
+
 	// Fires on WebSocket message
 	private onMessage(event: MessageEvent) {
+		if (event.data instanceof ArrayBuffer) {
+			this.onBinaryMessage(event.data);
+			return;
+		}
 		let msgArr: string[];
 		try {
 			msgArr = Guacutils.decode(event.data);
@@ -237,15 +271,7 @@ export default class CollabVMClient {
 				var x = parseInt(msgArr[3]);
 				var y = parseInt(msgArr[4]);
 				img.addEventListener('load', () => {
-					if (this.actualScreenSize.width !== this.canvasScale.width || this.actualScreenSize.height !== this.canvasScale.height)
+					this.loadRectangle(img, x, y);
-						this.unscaledCtx.drawImage(img, x, y);
-					// Scale the image to the canvas
-					this.ctx.drawImage(img, 0, 0, img.width, img.height, 
-						(x / this.actualScreenSize.width) * this.canvas.width, 
-						(y / this.actualScreenSize.height) * this.canvas.height, 
-						(img.width / this.actualScreenSize.width) * this.canvas.width, 
-						(img.height / this.actualScreenSize.height) * this.canvas.height
-					);
 				});
 				img.src = 'data:image/jpeg;base64,' + msgArr[5];
 				break;
@@ -426,6 +452,18 @@ export default class CollabVMClient {
 		}
 	}
 
+	private loadRectangle(img: HTMLImageElement, x: number, y: number) {
+		if (this.actualScreenSize.width !== this.canvasScale.width || this.actualScreenSize.height !== this.canvasScale.height)
+			this.unscaledCtx.drawImage(img, x, y);
+		// Scale the image to the canvas
+		this.ctx.drawImage(img, 0, 0, img.width, img.height,
+			(x / this.actualScreenSize.width) * this.canvas.width,
+			(y / this.actualScreenSize.height) * this.canvas.height,
+			(img.width / this.actualScreenSize.width) * this.canvas.width,
+			(img.height / this.actualScreenSize.height) * this.canvas.height
+		);
+	}
+
 	private onWindowResize(e: Event) {
 		if (!this.connectedToVM) return;
 		// If the canvas is the same size as the screen, don't bother redrawing
@@ -506,6 +544,7 @@ export default class CollabVMClient {
 			if (localStorage.getItem('collabvm-hide-flag') === 'true') this.send('noflag');
 			if (username === null) this.send('rename');
 			else this.send('rename', username);
+			if (DefaultCapabilities.length > 0) this.send('cap', ...DefaultCapabilities);
 			this.send('connect', id);
 			this.node = id;
 		});

src/ts/protocol/binaryprotocol/CollabVMCapabilities.ts

@@ -0,0 +1,8 @@
+export default class CollabVMCapabilities {
+    // Support for JPEG screen rects in binary msgpack format
+    bin: boolean;
+
+    constructor() {
+        this.bin = false;
+    }
+}

src/ts/protocol/binaryprotocol/CollabVMProtocolMessage.ts

@@ -0,0 +1,11 @@
+import CollabVMRectMessage from "./CollabVMRectMessage.js";
+
+export interface CollabVMProtocolMessage {
+    type: CollabVMProtocolMessageType;
+    rect?: CollabVMRectMessage | undefined;
+}
+
+export enum CollabVMProtocolMessageType {
+    // JPEG Dirty Rectangle
+    rect = 0,
+}

src/ts/protocol/binaryprotocol/CollabVMRectMessage.ts

@@ -0,0 +1,5 @@
+export default interface CollabVMRectMessage {
+    x: number;
+    y: number;
+    data: Uint8Array;
+}

yarn.lock

@@ -1511,6 +1511,11 @@
   dependencies:
     "@types/yargs-parser" "*"
 
+"@ygoe/msgpack@^1.0.3":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@ygoe/msgpack/-/msgpack-1.0.3.tgz#3889f4c0c2d68b2be83e1f6f4444efab02d6f257"
+  integrity sha512-Sjp0O/sNgOJxTOO1c2Zuu7nsHRIGu2iGPYyhUedKKbcHyUl73jbCaomEFJZHNb/6i94B+ZNZHVnFgpo0ENSXxQ==
+
 abortcontroller-polyfill@^1.1.9:
   version "1.7.5"
   resolved "https://registry.yarnpkg.com/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz#6738495f4e901fbb57b6c0611d0c75f76c485bed"