diff --git a/libopus.js b/libopus.js index 5b0ff4907..c18c0bd62 100644 --- a/libopus.js +++ b/libopus.js @@ -5550,6 +5550,7 @@ self.addEventListener('message', (e) => { dec = new Decoder(e.data.channels, e.data.sampleRate); } else { dec.input(e.data); - self.postMessage(dec.output().slice(0)); + var out = dec.output().slice(0); + self.postMessage(out, [out.buffer]); } }); \ No newline at end of file diff --git a/src/globals.js b/src/globals.js index dd96404f0..dcc40d43e 100644 --- a/src/globals.js +++ b/src/globals.js @@ -48,10 +48,38 @@ export function pushEvent(name, payload) { events.push(payload); } -let yuvWorker; +let yuvWorker = new Worker("./yuv.js"); +/* +let yuvCanvas; +let gl; +let pixels; +if (YUVCanvas.WebGLFrameSink.isAvailable()) { + var canvas = document.createElement('canvas'); + yuvCanvas = YUVCanvas.attach(canvas, { webGL: true }); + gl = canvas.getContext("webgl"); +} else { + yuvWorker = new Worker("./yuv.js"); +} +*/ export function draw(frame) { - if (yuvWorker) yuvWorker.postMessage(frame); + if (yuvWorker) { + // frame's (y/u/v).bytes already detached, can not transferrable any more. + yuvWorker.postMessage(frame); + } else { + var now = new Date().getTime(); + yuvCanvas.drawFrame(frame); + console.log(new Date().getTime() - now); + now = new Date().getTime(); + if (!pixels) pixels = new Uint8Array(canvas.width * canvas.height * 4); + gl.readPixels(0, 0, canvas.width, canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, pixels); + console.log(new Date().getTime() - now); + } +} + +export function sendOffCanvas(c) { + let canvas = c.transferControlToOffscreen(); + yuvWorker.postMessage({ canvas }, [canvas]); } export function setConn(conn) { @@ -280,7 +308,7 @@ function _getByName(name, arg) { return ''; } -let opusWorker; +let opusWorker = new Worker("./libopus.js"); let pcmPlayer; export function initAudio(channels, sampleRate) { @@ -289,14 +317,14 @@ export function initAudio(channels, sampleRate) { } export function playAudio(packet) { - opusWorker.postMessage(packet); + opusWorker.postMessage(packet, [packet.buffer]); } window.init = async () => { - yuvWorker = new Worker("./yuv.js"); - opusWorker = new Worker("./libopus.js"); - yuvWorker.onmessage = (e) => { - currentFrame = e.data; + if (yuvWorker) { + yuvWorker.onmessage = (e) => { + currentFrame = e.data; + } } opusWorker.onmessage = (e) => { pcmPlayer.feed(e.data); diff --git a/src/ui.js b/src/ui.js index b76824ba6..5e56fd93d 100644 --- a/src/ui.js +++ b/src/ui.js @@ -37,7 +37,8 @@ if (app) { id.value = localStorage.getItem('id'); const key = document.querySelector('#key'); key.value = localStorage.getItem('key'); - player = YUVCanvas.attach(document.getElementById('player')) + player = YUVCanvas.attach(document.getElementById('player')); + // globals.sendOffCanvas(document.getElementById('player')); }; window.connect = () => { @@ -51,6 +52,12 @@ if (app) { const conn = globals.newConn(); conn.setMsgbox(msgbox); conn.setDraw((f) => { + /* + if (!(document.getElementById('player').width > 0)) { + document.getElementById('player').width = f.format.displayWidth; + document.getElementById('player').height = f.format.displayHeight; + } + */ globals.draw(f); player.drawFrame(f); }); diff --git a/yuv.js b/yuv.js index 35feb485e..33ca9e3cb 100644 --- a/yuv.js +++ b/yuv.js @@ -1,3 +1,8 @@ +/* +var window = {}; +importScripts('./yuv-canvas-1.2.6.js'); +*/ + var wasmExports; fetch('yuv.wasm').then(function (res) { return res.arrayBuffer(); }) @@ -58,13 +63,27 @@ function I420ToARGB(yb) { } var currentFrame; +var canvas; +var yuvCanvas; self.addEventListener('message', (e) => { - currentFrame = e.data; + if (e.data.canvas) { + canvas = e.data.canvas; + yuvCanvas = window.YUVCanvas.attach(canvas, { webGL: true }); + } else { + if (yuvCanvas) { + var now = new Date().getTime(); + yuvCanvas.drawFrame(e.data); + console.log(new Date().getTime() - now); + } else { + currentFrame = e.data; + } + } }); function run() { if (currentFrame) { - self.postMessage(I420ToARGB(currentFrame)); + var data = I420ToARGB(currentFrame); + self.postMessage(data, [data.buffer]); currentFrame = undefined; } setTimeout(run, 1);