diff --git a/src/js/detect-worker.js b/src/js/detect-worker.js index da5b4c2..e34ca51 100644 --- a/src/js/detect-worker.js +++ b/src/js/detect-worker.js @@ -23,6 +23,14 @@ self.onconnect = (e) => { }) e.data.image.close() break + case 'videoFrame': + videoFrame(e.data.image).then((franeDet) =>{ + port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH}) + }).catch((err) => { + port.postMessage({error: true, message: err.message}) + }) + e.data.image.close() + break default: console.log('Worker message incoming:') console.log(e) @@ -207,63 +215,34 @@ function remoteTimeout () { f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') } -async function videoFrameDetect (vidData) { - await this.loadModel(this.miniLocation) +async function videoFrame (vidData) { const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - const imCanvas = this.$refs.image_cvs - const imageCtx = imCanvas.getContext("2d") - const target = this.$refs.target_image - await tf.nextFrame(); - imCanvas.width = imCanvas.clientWidth - imCanvas.height = imCanvas.clientHeight - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - var imgWidth - var imgHeight - const imgAspect = vidData.width / vidData.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - while (this.videoAvailable) { - console.time('frame-process') - try { - const input = tf.tidy(() => { - return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) - }) - const res = model.predict(input) - const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] + console.time('frame-process') + let rawCoords = [] + try { + const input = tf.tidy(() => { + return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) + }) + const res = model.predict(input) + const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] - let rawCoords = [] - if (rawRes) { - for (var i = 0; i < rawRes.length; i++) { - let getScores = rawRes[i].slice(4) - if (getScores.some( s => s > .5)) { - let foundTarget = rawRes[i].slice(0,2) - foundTarget.push(Math.max(...getScores)) - rawCoords.push(foundTarget) - } - } - - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - for (var coord of rawCoords) { - console.log(`x: ${coord[0]}, y: ${coord[1]}`) - let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5 - let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5 - imageCtx.globalAlpha = coord[2] - imageCtx.drawImage(target, pointX, pointY, 20, 20) + if (rawRes) { + for (var i = 0; i < rawRes.length; i++) { + let getScores = rawRes[i].slice(4) + if (getScores.some( s => s > .5)) { + let foundTarget = rawRes[i].slice(0,2) + foundTarget.push(Math.max(...getScores)) + rawCoords.push(foundTarget) } } - tf.dispose(input) - tf.dispose(res) - tf.dispose(rawRes) - } catch (e) { - console.log(e) + } - console.timeEnd('frame-process') - await tf.nextFrame(); + tf.dispose(input) + tf.dispose(res) + tf.dispose(rawRes) + } catch (e) { + console.log(e) } + console.timeEnd('frame-process') + return {cds: rawCoords, mW: modelWidth, mH: modelHeight} } \ No newline at end of file diff --git a/src/pages/camera-mixin.js b/src/pages/camera-mixin.js index 314f596..d107415 100644 --- a/src/pages/camera-mixin.js +++ b/src/pages/camera-mixin.js @@ -1,3 +1,5 @@ +import { f7 } from 'framework7-vue' + export default { methods: { async openCamera(imContain) { @@ -38,6 +40,51 @@ export default { const tempCtx = tempCVS.getContext('2d') tempCtx.drawImage(vidViewer, 0, 0) this.getImage(tempCVS.toDataURL()) + }, + async videoFrameDetect (vidData) { + const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + vidWorker.port.onmessage = (eVid) => { + self = this + if (eVid.data.error) { + console.log(eVid.data.message) + f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`) + } else if (this.videoAvailable) { + createImageBitmap(vidData).then(imVideoFrame => { + vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) + }) + if (eVid.data.coords) { + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + for (var coord of eVid.data.coords) { + let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / eVid.data.modelWidth) * imgWidth - 10 + let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / eVid.data.modelHeight) * imgHeight - 10 + console.debug(`cx: ${pointX}, cy: ${pointY}`) + imageCtx.globalAlpha = coord[2] + imageCtx.drawImage(target, pointX, pointY, 20, 20) + } + } + } + } + + vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true}) + const imCanvas = this.$refs.image_cvs + const imageCtx = imCanvas.getContext("2d") + const target = this.$refs.target_image + var imgWidth + var imgHeight + f7.utils.nextFrame(() => { + imCanvas.width = imCanvas.clientWidth + imCanvas.height = imCanvas.clientHeight + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + const imgAspect = vidData.width / vidData.height + const rendAspect = imCanvas.width / imCanvas.height + if (imgAspect >= rendAspect) { + imgWidth = imCanvas.width + imgHeight = imCanvas.width / imgAspect + } else { + imgWidth = imCanvas.height * imgAspect + imgHeight = imCanvas.height + } + }) } } } \ No newline at end of file diff --git a/src/pages/detection-mixin.js b/src/pages/detection-mixin.js index d0b7197..2bffbc5 100644 --- a/src/pages/detection-mixin.js +++ b/src/pages/detection-mixin.js @@ -172,65 +172,5 @@ export default { this.detecting = false f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') }, - async videoFrameDetect (vidData) { - await this.loadModel(this.miniLocation) - const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - const imCanvas = this.$refs.image_cvs - const imageCtx = imCanvas.getContext("2d") - const target = this.$refs.target_image - await tf.nextFrame(); - imCanvas.width = imCanvas.clientWidth - imCanvas.height = imCanvas.clientHeight - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - var imgWidth - var imgHeight - const imgAspect = vidData.width / vidData.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - while (this.videoAvailable) { - console.time('frame-process') - try { - const input = tf.tidy(() => { - return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) - }) - const res = model.predict(input) - const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] - - let rawCoords = [] - if (rawRes) { - for (var i = 0; i < rawRes.length; i++) { - let getScores = rawRes[i].slice(4) - if (getScores.some( s => s > .5)) { - let foundTarget = rawRes[i].slice(0,2) - foundTarget.push(Math.max(...getScores)) - rawCoords.push(foundTarget) - } - } - - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - for (var coord of rawCoords) { - console.log(`x: ${coord[0]}, y: ${coord[1]}`) - let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5 - let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5 - imageCtx.globalAlpha = coord[2] - imageCtx.drawImage(target, pointX, pointY, 20, 20) - } - } - tf.dispose(input) - tf.dispose(res) - tf.dispose(rawRes) - } catch (e) { - console.log(e) - } - console.timeEnd('frame-process') - await tf.nextFrame(); - } - } } } \ No newline at end of file