From 1a703b0100ec7cb2f3eb04b66fd29c9e5be5aec8 Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Mon, 29 Jul 2024 00:54:15 +0000 Subject: [PATCH] Switch shared worker to basic service worker (#191) Shared workers seem to cause problems with iOS (and sharing wasn't really required anyway), so this PR changes the shared workers to non-shared workers. As a benefit, it preloads the full model and video models simultaneously which iproves performance when starting the video and running post video detection. Signed-off-by: Justin Georgi Reviewed-on: https://gitea.azgeorgis.net/ALVINN/ALVINN_f7/pulls/191 --- src/assets/detect-worker.js | 68 +++++++++++++++++-------------------- src/pages/camera-mixin.js | 15 ++++---- src/pages/detect.vue | 36 +++++++++++++------- 3 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/assets/detect-worker.js b/src/assets/detect-worker.js index 828d1d4..f220559 100644 --- a/src/assets/detect-worker.js +++ b/src/assets/detect-worker.js @@ -1,45 +1,39 @@ import * as tf from '@tensorflow/tfjs' -import { f7 } from 'framework7-vue' let model = null -self.onconnect = (e) => { - const port = e.ports[0]; - - port.onmessage = function (e) { - switch (e.data.call) { - case 'loadModel': - loadModel('.' + e.data.weights,e.data.preload).then(() => { - port.postMessage({success: 'model'}) - }).catch((err) => { - port.postMessage({error: true, message: err.message}) - }) - break - case 'localDetect': - localDetect(e.data.image).then((dets) => { - port.postMessage({success: 'detection', detections: dets}) - }).catch((err) => { - port.postMessage({error: true, message: err.message}) - }) - e.data.image.close() - break - case 'videoFrame': - videoFrame(e.data.image).then((frameDet) =>{ - port.postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH}) - }).catch((err) => { - port.postMessage({error: true, message: err.message}) - }) - e.data.image.close() - break - default: - console.log('Worker message incoming:') - console.log(e) - port.postMessage({result1: 'First result', result2: 'Second result'}) - break - } +onmessage = function (e) { + switch (e.data.call) { + case 'loadModel': + loadModel('.' + e.data.weights,e.data.preload).then(() => { + postMessage({success: 'model'}) + }).catch((err) => { + postMessage({error: true, message: err.message}) + }) + break + case 'localDetect': + localDetect(e.data.image).then((dets) => { + postMessage({success: 'detection', detections: dets}) + }).catch((err) => { + //throw (err) + postMessage({error: true, message: err.message}) + }) + e.data.image.close() + break + case 'videoFrame': + videoFrame(e.data.image).then((frameDet) =>{ + postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH}) + }).catch((err) => { + postMessage({error: true, message: err.message}) + }) + e.data.image.close() + break + default: + console.log('Worker message incoming:') + console.log(e) + postMessage({result1: 'First result', result2: 'Second result'}) + break } - - port.start() } async function loadModel(weights, preload) { diff --git a/src/pages/camera-mixin.js b/src/pages/camera-mixin.js index 9333170..412c1e8 100644 --- a/src/pages/camera-mixin.js +++ b/src/pages/camera-mixin.js @@ -1,7 +1,5 @@ import { f7 } from 'framework7-vue' -import detectionWorker from '../assets/detect-worker.js?sharedworker' - export default { methods: { async openCamera(imContain) { @@ -44,15 +42,19 @@ export default { this.getImage(tempCVS.toDataURL()) }, async videoFrameDetect (vidData) { - const vidWorker = new detectionWorker() - vidWorker.port.onmessage = (eVid) => { - self = this + const startDetection = () => { + createImageBitmap(vidData).then(imVideoFrame => { + this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) + }) + } + vidData.addEventListener('resize',startDetection,{once: true}) + this.vidWorker.onmessage = (eVid) => { if (eVid.data.error) { console.log(eVid.data.message) f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`) } else if (this.videoAvailable) { createImageBitmap(vidData).then(imVideoFrame => { - vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) + this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) }) if (eVid.data.coords) { imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) @@ -67,7 +69,6 @@ export default { } } - vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true}) const imCanvas = this.$refs.image_cvs const imageCtx = imCanvas.getContext("2d") const target = this.$refs.target_image diff --git a/src/pages/detect.vue b/src/pages/detect.vue index 3fca1ae..d6fe581 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -140,7 +140,7 @@ import detectionMixin from './detection-mixin' import cameraMixin from './camera-mixin' - import detectionWorker from '../assets/detect-worker.js?sharedworker' + import detectionWorker from '../assets/detect-worker.js?worker' export default { mixins: [submitMixin, detectionMixin, cameraMixin], @@ -179,7 +179,9 @@ videoDeviceAvailable: false, videoAvailable: false, cameraStream: null, - infoLinkPos: {} + infoLinkPos: {}, + detectWorker: null, + vidWorker: null } }, setup() { @@ -216,8 +218,8 @@ if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings) }, mounted () { - const mountWorker = new detectionWorker() - mountWorker.port.onmessage = (eMount) => { + this.detectWorker = new detectionWorker() + this.detectWorker.onmessage = (eMount) => { self = this if (eMount.data.error) { console.log(eMount.data.message) @@ -225,13 +227,22 @@ } self.modelLoading = false } + this.vidWorker = new detectionWorker() + this.vidWorker.onmessage = (eMount) => { + self = this + if (eMount.data.error) { + console.log(eMount.data.message) + f7.dialog.alert(`ALVINN AI nano model error: ${eMount.data.message}`) + } + } if (this.serverSettings && this.serverSettings.use) { this.getRemoteLabels() this.modelLoading = false } else { this.modelLoading = true - mountWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true}) + this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true}) + this.vidWorker.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true}) } window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') } }, @@ -293,12 +304,13 @@ return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)` }, async setData () { - const detectWorker = new detectionWorker() - detectWorker.port.onmessage = (eDetect) => { + //const detectWorker = new detectionWorker() + this.detectWorker.onmessage = (eDetect) => { self = this if (eDetect.data.error) { self.detecting = false self.resultData = {} + loadFailure() f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`) } else if (eDetect.data.success == 'detection') { self.detecting = false @@ -308,8 +320,8 @@ } self.uploadDirty = true } else if (eDetect.data.success == 'model') { - this.reloadModel = false - loadSuccess(true) + self.reloadModel = false + loadSuccess() } } @@ -319,9 +331,9 @@ loadSuccess = res loadFailure = rej if (this.reloadModel) { - detectWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation}) + this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation}) } else { - loadSuccess(true) + loadSuccess() } }) @@ -329,7 +341,7 @@ this.remoteDetect() } else { Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => { - detectWorker.port.postMessage({call: 'localDetect', image: res[1]}, [res[1]]) + this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]]) }) } },