try out webgl canvas, draw fast, but copy pixels out slow

libopus.js

@@ -5550,6 +5550,7 @@ self.addEventListener('message', (e) => {
     dec = new Decoder(e.data.channels, e.data.sampleRate);
   } else {
     dec.input(e.data);
-    self.postMessage(dec.output().slice(0));
+    var out = dec.output().slice(0);
+    self.postMessage(out, [out.buffer]);
   }
 });

src/globals.js

@@ -48,10 +48,38 @@ export function pushEvent(name, payload) {
   events.push(payload);
 }
 
-let yuvWorker;
+let yuvWorker = new Worker("./yuv.js");
+/*
+let yuvCanvas;
+let gl;
+let pixels;
+if (YUVCanvas.WebGLFrameSink.isAvailable()) {
+  var canvas = document.createElement('canvas');
+  yuvCanvas = YUVCanvas.attach(canvas, { webGL: true });
+  gl = canvas.getContext("webgl");
+} else {
+  yuvWorker = new Worker("./yuv.js");
+}
+*/
 
 export function draw(frame) {
-  if (yuvWorker) yuvWorker.postMessage(frame);
+  if (yuvWorker) {
+    // frame's (y/u/v).bytes already detached, can not transferrable any more.
+    yuvWorker.postMessage(frame);
+  } else {
+    var now = new Date().getTime();
+    yuvCanvas.drawFrame(frame);
+    console.log(new Date().getTime() - now);
+    now = new Date().getTime();
+    if (!pixels) pixels = new Uint8Array(canvas.width * canvas.height * 4);
+    gl.readPixels(0, 0, canvas.width, canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
+    console.log(new Date().getTime() - now);
+  }
+}
+
+export function sendOffCanvas(c) {
+  let canvas = c.transferControlToOffscreen();
+  yuvWorker.postMessage({ canvas }, [canvas]);
 }
 
 export function setConn(conn) {
@@ -280,7 +308,7 @@ function _getByName(name, arg) {
   return '';
 }
 
-let opusWorker;
+let opusWorker = new Worker("./libopus.js");
 let pcmPlayer;
 
 export function initAudio(channels, sampleRate) {
@@ -289,15 +317,15 @@ export function initAudio(channels, sampleRate) {
 }
 
 export function playAudio(packet) {
-  opusWorker.postMessage(packet);
+  opusWorker.postMessage(packet, [packet.buffer]);
 }
 
 window.init = async () => {
-  yuvWorker = new Worker("./yuv.js");
+  if (yuvWorker) {
-  opusWorker = new Worker("./libopus.js");
     yuvWorker.onmessage = (e) => {
       currentFrame = e.data;
     }
+  }
   opusWorker.onmessage = (e) => {
     pcmPlayer.feed(e.data);
   }

src/ui.js

@@ -37,7 +37,8 @@ if (app) {
     id.value = localStorage.getItem('id');
     const key = document.querySelector('#key');
     key.value = localStorage.getItem('key');
-    player = YUVCanvas.attach(document.getElementById('player'))
+    player = YUVCanvas.attach(document.getElementById('player'));
+    // globals.sendOffCanvas(document.getElementById('player'));
   };
 
   window.connect = () => {
@@ -51,6 +52,12 @@ if (app) {
       const conn = globals.newConn();
       conn.setMsgbox(msgbox);
       conn.setDraw((f) => {
+        /*
+        if (!(document.getElementById('player').width > 0)) {
+          document.getElementById('player').width = f.format.displayWidth;
+          document.getElementById('player').height = f.format.displayHeight;
+        }
+        */
         globals.draw(f);
         player.drawFrame(f);
       });

yuv.js

@@ -1,3 +1,8 @@
+/*
+var window = {};
+importScripts('./yuv-canvas-1.2.6.js');
+*/
+
 var wasmExports;
 
 fetch('yuv.wasm').then(function (res) { return res.arrayBuffer(); })
@@ -58,13 +63,27 @@ function I420ToARGB(yb) {
 }
 
 var currentFrame;
+var canvas;
+var yuvCanvas;
 self.addEventListener('message', (e) => {
+  if (e.data.canvas) {
+    canvas = e.data.canvas;
+    yuvCanvas = window.YUVCanvas.attach(canvas, { webGL: true });
+  } else {
+    if (yuvCanvas) {
+      var now = new Date().getTime();
+      yuvCanvas.drawFrame(e.data);
+      console.log(new Date().getTime() - now);
+    } else {
       currentFrame = e.data;
+    }
+  }
 });
 
 function run() {
   if (currentFrame) {
-    self.postMessage(I420ToARGB(currentFrame));
+    var data = I420ToARGB(currentFrame);
+    self.postMessage(data, [data.buffer]);
     currentFrame = undefined;
   }
   setTimeout(run, 1);