From c7622c36028b9e28202aa2df9964263e66fd7a23 Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Thu, 18 Jul 2024 20:51:27 -0700 Subject: [PATCH 1/4] Add basic worker function Signed-off-by: Justin Georgi --- src/js/detect-worker.js | 260 ++++++++++++++++++++++++++++++++++++++++ src/pages/detect.vue | 29 +++-- 2 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 src/js/detect-worker.js diff --git a/src/js/detect-worker.js b/src/js/detect-worker.js new file mode 100644 index 0000000..97b154e --- /dev/null +++ b/src/js/detect-worker.js @@ -0,0 +1,260 @@ +import * as tf from '@tensorflow/tfjs' +import { f7 } from 'framework7-vue' + +let model = null + +self.onconnect = (e) => { + const port = e.ports[0]; + + port.onmessage = function (e) { + switch (e.data.call) { + case 'loadModel': + loadModel('.' + e.data.weights,e.data.preload).then(() => { + port.postMessage({success: true}) + }).catch((err) => { + port.postMessage({error: true, message: err.message}) + }) + break + default: + console.log('Worker message incoming:') + console.log(e) + port.postMessage({result1: 'First result', result2: 'Second result'}) + break + } + } + + port.start() +} + +async function loadModel(weights, preload) { + if (model && model.modelURL == weights) { + return model + } else if (model) { + tf.dispose(model) + } + model = await tf.loadGraphModel(weights) + const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) + /***************** + * If preloading then run model + * once on fake data to preload + * weights for a faster response + *****************/ + if (preload) { + const dummyT = tf.ones([1,modelWidth,modelHeight,3]) + model.predict(dummyT) + } + return model +} + +async function localDetect(imageData) { + console.time('pre-process') + const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) + let gTense = null + const input = tf.tidy(() => { + gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0) + return tf.concat([gTense,gTense,gTense],3) + }) + tf.dispose(gTense) + console.timeEnd('pre-process') + + console.time('run prediction') + const res = model.predict(input) + const tRes = tf.transpose(res,[0,2,1]) + const rawRes = tRes.arraySync()[0] + console.timeEnd('run prediction') + + console.time('post-process') + const outputSize = res.shape[1] + let rawBoxes = [] + let rawScores = [] + + for (var i = 0; i < rawRes.length; i++) { + var getScores = rawRes[i].slice(4) + if (getScores.every( s => s < .05)) { continue } + var getBox = rawRes[i].slice(0,4) + var boxCalc = [ + (getBox[0] - (getBox[2] / 2)) / modelWidth, + (getBox[1] - (getBox[3] / 2)) / modelHeight, + (getBox[0] + (getBox[2] / 2)) / modelWidth, + (getBox[1] + (getBox[3] / 2)) / modelHeight, + ] + rawBoxes.push(boxCalc) + rawScores.push(getScores) + } + + if (rawBoxes.length > 0) { + const tBoxes = tf.tensor2d(rawBoxes) + let tScores = null + let resBoxes = null + let validBoxes = [] + let structureScores = null + let boxes_data = [] + let scores_data = [] + let classes_data = [] + for (var c = 0; c < outputSize - 4; c++) { + structureScores = rawScores.map(x => x[c]) + tScores = tf.tensor1d(structureScores) + resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05) + validBoxes = resBoxes.dataSync() + tf.dispose(resBoxes) + if (validBoxes) { + boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx))) + var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx)) + scores_data.push(...outputScores) + classes_data.push(...outputScores.fill(c)) + } + } + + validBoxes = [] + tf.dispose(tBoxes) + tf.dispose(tScores) + tf.dispose(tRes) + const valid_detections_data = classes_data.length + var output = { + detections: [] + } + for (var i =0; i < valid_detections_data; i++) { + var [dLeft, dTop, dRight, dBottom] = boxes_data[i] + output.detections.push({ + "top": dTop, + "left": dLeft, + "bottom": dBottom, + "right": dRight, + "label": this.detectorLabels[classes_data[i]].name, + "confidence": scores_data[i] * 100 + }) + } + } + tf.dispose(res) + tf.dispose(input) + console.timeEnd('post-process') + + return output || { detections: [] } +} + +function getRemoteLabels() { + var self = this + var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors` + var xhr = new XMLHttpRequest() + xhr.open("GET", modelURL) + xhr.setRequestHeader('Content-Type', 'application/json') + xhr.timeout = 10000 + xhr.ontimeout = this.remoteTimeout + xhr.onload = function () { + if (this.status !== 200) { + console.log(xhr.response) + const errorResponse = JSON.parse(xhr.response) + f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`) + return + } + var detectors = JSON.parse(xhr.response).detectors + var findLabel = detectors + .find( d => { return d.name == self.detectorName } )?.labels + .filter( l => { return l != "" } ).sort() + .map( l => { return {'name': l, 'detect': true} } ) + self.detectorLabels = findLabel || [] + } + xhr.onerror = function (e) { + f7.dialog.alert('ALVINN has encountered an unknown server error') + return + } + + xhr.send() +} + +function remoteDetect() { + var self = this + var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect` + var xhr = new XMLHttpRequest() + xhr.open("POST", modelURL) + xhr.timeout = 10000 + xhr.ontimeout = this.remoteTimeout + xhr.setRequestHeader('Content-Type', 'application/json') + xhr.onload = function () { + self.detecting = false + if (this.status !== 200) { + console.log(xhr.response) + const errorResponse = JSON.parse(xhr.response) + f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`) + return; + } + self.resultData = JSON.parse(xhr.response) + self.uploadDirty = true + } + + var doodsData = { + "detector_name": this.detectorName, + "detect": { + "*": 1 + }, + "data": this.imageView.src.split(',')[1] + } + + xhr.send(JSON.stringify(doodsData)) +} + +function remoteTimeout () { + this.detecting = false + f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') +} + +async function videoFrameDetect (vidData) { + await this.loadModel(this.miniLocation) + const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) + const imCanvas = this.$refs.image_cvs + const imageCtx = imCanvas.getContext("2d") + const target = this.$refs.target_image + await tf.nextFrame(); + imCanvas.width = imCanvas.clientWidth + imCanvas.height = imCanvas.clientHeight + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + var imgWidth + var imgHeight + const imgAspect = vidData.width / vidData.height + const rendAspect = imCanvas.width / imCanvas.height + if (imgAspect >= rendAspect) { + imgWidth = imCanvas.width + imgHeight = imCanvas.width / imgAspect + } else { + imgWidth = imCanvas.height * imgAspect + imgHeight = imCanvas.height + } + while (this.videoAvailable) { + console.time('frame-process') + try { + const input = tf.tidy(() => { + return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) + }) + const res = model.predict(input) + const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] + + let rawCoords = [] + if (rawRes) { + for (var i = 0; i < rawRes.length; i++) { + let getScores = rawRes[i].slice(4) + if (getScores.some( s => s > .5)) { + let foundTarget = rawRes[i].slice(0,2) + foundTarget.push(Math.max(...getScores)) + rawCoords.push(foundTarget) + } + } + + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + for (var coord of rawCoords) { + console.log(`x: ${coord[0]}, y: ${coord[1]}`) + let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5 + let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5 + imageCtx.globalAlpha = coord[2] + imageCtx.drawImage(target, pointX, pointY, 20, 20) + } + } + tf.dispose(input) + tf.dispose(res) + tf.dispose(rawRes) + } catch (e) { + console.log(e) + } + console.timeEnd('frame-process') + await tf.nextFrame(); + } +} \ No newline at end of file diff --git a/src/pages/detect.vue b/src/pages/detect.vue index 8e83f6a..7520fdc 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -214,18 +214,33 @@ if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings) }, mounted () { + const mountWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + mountWorker.port.onmessage = (eMount) => { + self = this + if (eMount.data.error) { + console.log(eMount.data.message) + f7.dialog.alert(`ALVINN AI model error: ${eMount.data.message}`) + } + console.log(eMount) + console.log('Model loading complete.') + console.log(self.imageLoadMode) + self.modelLoading = false + } + if (this.serverSettings && this.serverSettings.use) { this.getRemoteLabels() this.modelLoading = false } else { this.modelLoading = true - this.loadModel(this.modelLocation, true).then(() => { - this.modelLoading = false - }).catch((e) => { - console.log(e.message) - f7.dialog.alert(`ALVINN AI model error: ${e.message}`) - this.modelLoading = false - }) + console.log('Loading model...') + mountWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true}) +// this.loadModel(this.modelLocation, true).then(() => { +// this.modelLoading = false +// }).catch((e) => { +// console.log(e.message) +// f7.dialog.alert(`ALVINN AI model error: ${e.message}`) +// this.modelLoading = false +// }) } window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') } }, -- 2.49.1 From cd6cad9e7323678eb1293ee36301ef0abba77fe6 Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Tue, 23 Jul 2024 14:18:49 -0700 Subject: [PATCH 2/4] Add shared worker structure detection Signed-off-by: Justin Georgi --- src/js/detect-worker.js | 11 ++++++++++- src/pages/detect.vue | 43 ++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/js/detect-worker.js b/src/js/detect-worker.js index 97b154e..da5b4c2 100644 --- a/src/js/detect-worker.js +++ b/src/js/detect-worker.js @@ -15,6 +15,14 @@ self.onconnect = (e) => { port.postMessage({error: true, message: err.message}) }) break + case 'localDetect': + localDetect(e.data.image).then((dets) => { + port.postMessage({success: true, detections: dets}) + }).catch((err) => { + port.postMessage({error: true, message: err.message}) + }) + e.data.image.close() + break default: console.log('Worker message incoming:') console.log(e) @@ -120,7 +128,8 @@ async function localDetect(imageData) { "left": dLeft, "bottom": dBottom, "right": dRight, - "label": this.detectorLabels[classes_data[i]].name, +// "label": this.detectorLabels[classes_data[i]].name, + "label": classes_data[i], "confidence": scores_data[i] * 100 }) } diff --git a/src/pages/detect.vue b/src/pages/detect.vue index 7520fdc..d901a54 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -221,9 +221,6 @@ console.log(eMount.data.message) f7.dialog.alert(`ALVINN AI model error: ${eMount.data.message}`) } - console.log(eMount) - console.log('Model loading complete.') - console.log(self.imageLoadMode) self.modelLoading = false } @@ -232,15 +229,7 @@ this.modelLoading = false } else { this.modelLoading = true - console.log('Loading model...') mountWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true}) -// this.loadModel(this.modelLocation, true).then(() => { -// this.modelLoading = false -// }).catch((e) => { -// console.log(e.message) -// f7.dialog.alert(`ALVINN AI model error: ${e.message}`) -// this.modelLoading = false -// }) } window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') } }, @@ -302,6 +291,23 @@ return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)` }, async setData () { + const detectWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + detectWorker.port.onmessage = (eDetect) => { + self = this + if (eDetect.data.error) { + self.detecting = false + self.resultData = {} + f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`) + } else { + self.detecting = false + self.resultData = eDetect.data.detections + if (self.resultData) { + self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name}) + } + self.uploadDirty = true + } + } + if (this.reloadModel) { await this.loadModel(this.modelLocation) this.reloadModel = false @@ -309,15 +315,8 @@ if (this.serverSettings && this.serverSettings.use) { this.remoteDetect() } else { - this.localDetect(this.imageView).then(dets => { - this.detecting = false - this.resultData = dets - this.uploadDirty = true - }).catch((e) => { - console.log(e.message) - this.detecting = false - this.resultData = {} - f7.dialog.alert(`ALVINN structure finding error: ${e.message}`) + createImageBitmap(this.imageView).then(imData => { + detectWorker.port.postMessage({call: 'localDetect', image: imData}, [imData]) }) } }, @@ -464,9 +463,9 @@ * setTimeout is not a good solution, but it's the only way * I can find to not cut off drawing of the canvas background ******/ - setTimeout(() => { +// setTimeout(() => { this.setData() - }, 1) +// }, 1) }).catch((e) => { console.log(e.message) f7.dialog.alert(`Error loading image: ${e.message}`) -- 2.49.1 From 77b108af6cdc9757e7c964a70292444089c0e6a7 Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Wed, 24 Jul 2024 12:00:48 -0700 Subject: [PATCH 3/4] Add worker video detection Signed-off-by: Justin Georgi --- src/js/detect-worker.js | 85 ++++++++++++++---------------------- src/pages/camera-mixin.js | 47 ++++++++++++++++++++ src/pages/detection-mixin.js | 60 ------------------------- 3 files changed, 79 insertions(+), 113 deletions(-) diff --git a/src/js/detect-worker.js b/src/js/detect-worker.js index da5b4c2..e34ca51 100644 --- a/src/js/detect-worker.js +++ b/src/js/detect-worker.js @@ -23,6 +23,14 @@ self.onconnect = (e) => { }) e.data.image.close() break + case 'videoFrame': + videoFrame(e.data.image).then((franeDet) =>{ + port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH}) + }).catch((err) => { + port.postMessage({error: true, message: err.message}) + }) + e.data.image.close() + break default: console.log('Worker message incoming:') console.log(e) @@ -207,63 +215,34 @@ function remoteTimeout () { f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') } -async function videoFrameDetect (vidData) { - await this.loadModel(this.miniLocation) +async function videoFrame (vidData) { const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - const imCanvas = this.$refs.image_cvs - const imageCtx = imCanvas.getContext("2d") - const target = this.$refs.target_image - await tf.nextFrame(); - imCanvas.width = imCanvas.clientWidth - imCanvas.height = imCanvas.clientHeight - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - var imgWidth - var imgHeight - const imgAspect = vidData.width / vidData.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - while (this.videoAvailable) { - console.time('frame-process') - try { - const input = tf.tidy(() => { - return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) - }) - const res = model.predict(input) - const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] + console.time('frame-process') + let rawCoords = [] + try { + const input = tf.tidy(() => { + return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) + }) + const res = model.predict(input) + const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] - let rawCoords = [] - if (rawRes) { - for (var i = 0; i < rawRes.length; i++) { - let getScores = rawRes[i].slice(4) - if (getScores.some( s => s > .5)) { - let foundTarget = rawRes[i].slice(0,2) - foundTarget.push(Math.max(...getScores)) - rawCoords.push(foundTarget) - } - } - - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - for (var coord of rawCoords) { - console.log(`x: ${coord[0]}, y: ${coord[1]}`) - let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5 - let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5 - imageCtx.globalAlpha = coord[2] - imageCtx.drawImage(target, pointX, pointY, 20, 20) + if (rawRes) { + for (var i = 0; i < rawRes.length; i++) { + let getScores = rawRes[i].slice(4) + if (getScores.some( s => s > .5)) { + let foundTarget = rawRes[i].slice(0,2) + foundTarget.push(Math.max(...getScores)) + rawCoords.push(foundTarget) } } - tf.dispose(input) - tf.dispose(res) - tf.dispose(rawRes) - } catch (e) { - console.log(e) + } - console.timeEnd('frame-process') - await tf.nextFrame(); + tf.dispose(input) + tf.dispose(res) + tf.dispose(rawRes) + } catch (e) { + console.log(e) } + console.timeEnd('frame-process') + return {cds: rawCoords, mW: modelWidth, mH: modelHeight} } \ No newline at end of file diff --git a/src/pages/camera-mixin.js b/src/pages/camera-mixin.js index 314f596..d107415 100644 --- a/src/pages/camera-mixin.js +++ b/src/pages/camera-mixin.js @@ -1,3 +1,5 @@ +import { f7 } from 'framework7-vue' + export default { methods: { async openCamera(imContain) { @@ -38,6 +40,51 @@ export default { const tempCtx = tempCVS.getContext('2d') tempCtx.drawImage(vidViewer, 0, 0) this.getImage(tempCVS.toDataURL()) + }, + async videoFrameDetect (vidData) { + const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + vidWorker.port.onmessage = (eVid) => { + self = this + if (eVid.data.error) { + console.log(eVid.data.message) + f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`) + } else if (this.videoAvailable) { + createImageBitmap(vidData).then(imVideoFrame => { + vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) + }) + if (eVid.data.coords) { + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + for (var coord of eVid.data.coords) { + let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / eVid.data.modelWidth) * imgWidth - 10 + let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / eVid.data.modelHeight) * imgHeight - 10 + console.debug(`cx: ${pointX}, cy: ${pointY}`) + imageCtx.globalAlpha = coord[2] + imageCtx.drawImage(target, pointX, pointY, 20, 20) + } + } + } + } + + vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true}) + const imCanvas = this.$refs.image_cvs + const imageCtx = imCanvas.getContext("2d") + const target = this.$refs.target_image + var imgWidth + var imgHeight + f7.utils.nextFrame(() => { + imCanvas.width = imCanvas.clientWidth + imCanvas.height = imCanvas.clientHeight + imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) + const imgAspect = vidData.width / vidData.height + const rendAspect = imCanvas.width / imCanvas.height + if (imgAspect >= rendAspect) { + imgWidth = imCanvas.width + imgHeight = imCanvas.width / imgAspect + } else { + imgWidth = imCanvas.height * imgAspect + imgHeight = imCanvas.height + } + }) } } } \ No newline at end of file diff --git a/src/pages/detection-mixin.js b/src/pages/detection-mixin.js index d0b7197..2bffbc5 100644 --- a/src/pages/detection-mixin.js +++ b/src/pages/detection-mixin.js @@ -172,65 +172,5 @@ export default { this.detecting = false f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') }, - async videoFrameDetect (vidData) { - await this.loadModel(this.miniLocation) - const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - const imCanvas = this.$refs.image_cvs - const imageCtx = imCanvas.getContext("2d") - const target = this.$refs.target_image - await tf.nextFrame(); - imCanvas.width = imCanvas.clientWidth - imCanvas.height = imCanvas.clientHeight - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - var imgWidth - var imgHeight - const imgAspect = vidData.width / vidData.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - while (this.videoAvailable) { - console.time('frame-process') - try { - const input = tf.tidy(() => { - return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) - }) - const res = model.predict(input) - const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0] - - let rawCoords = [] - if (rawRes) { - for (var i = 0; i < rawRes.length; i++) { - let getScores = rawRes[i].slice(4) - if (getScores.some( s => s > .5)) { - let foundTarget = rawRes[i].slice(0,2) - foundTarget.push(Math.max(...getScores)) - rawCoords.push(foundTarget) - } - } - - imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height) - for (var coord of rawCoords) { - console.log(`x: ${coord[0]}, y: ${coord[1]}`) - let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5 - let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5 - imageCtx.globalAlpha = coord[2] - imageCtx.drawImage(target, pointX, pointY, 20, 20) - } - } - tf.dispose(input) - tf.dispose(res) - tf.dispose(rawRes) - } catch (e) { - console.log(e) - } - console.timeEnd('frame-process') - await tf.nextFrame(); - } - } } } \ No newline at end of file -- 2.49.1 From 205f2da87dd6e79eae4cd2e6c3795743f91035dd Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Thu, 25 Jul 2024 10:54:02 -0700 Subject: [PATCH 4/4] Clean up shared worker addition Signed-off-by: Justin Georgi --- src/{js => assets}/detect-worker.js | 74 ++----------------- src/pages/camera-mixin.js | 2 +- src/pages/detect.vue | 34 ++++++--- src/pages/detection-mixin.js | 107 ---------------------------- 4 files changed, 28 insertions(+), 189 deletions(-) rename src/{js => assets}/detect-worker.js (69%) diff --git a/src/js/detect-worker.js b/src/assets/detect-worker.js similarity index 69% rename from src/js/detect-worker.js rename to src/assets/detect-worker.js index e34ca51..828d1d4 100644 --- a/src/js/detect-worker.js +++ b/src/assets/detect-worker.js @@ -10,22 +10,22 @@ self.onconnect = (e) => { switch (e.data.call) { case 'loadModel': loadModel('.' + e.data.weights,e.data.preload).then(() => { - port.postMessage({success: true}) + port.postMessage({success: 'model'}) }).catch((err) => { port.postMessage({error: true, message: err.message}) }) break case 'localDetect': localDetect(e.data.image).then((dets) => { - port.postMessage({success: true, detections: dets}) + port.postMessage({success: 'detection', detections: dets}) }).catch((err) => { port.postMessage({error: true, message: err.message}) }) e.data.image.close() break case 'videoFrame': - videoFrame(e.data.image).then((franeDet) =>{ - port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH}) + videoFrame(e.data.image).then((frameDet) =>{ + port.postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH}) }).catch((err) => { port.postMessage({error: true, message: err.message}) }) @@ -149,72 +149,6 @@ async function localDetect(imageData) { return output || { detections: [] } } -function getRemoteLabels() { - var self = this - var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors` - var xhr = new XMLHttpRequest() - xhr.open("GET", modelURL) - xhr.setRequestHeader('Content-Type', 'application/json') - xhr.timeout = 10000 - xhr.ontimeout = this.remoteTimeout - xhr.onload = function () { - if (this.status !== 200) { - console.log(xhr.response) - const errorResponse = JSON.parse(xhr.response) - f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`) - return - } - var detectors = JSON.parse(xhr.response).detectors - var findLabel = detectors - .find( d => { return d.name == self.detectorName } )?.labels - .filter( l => { return l != "" } ).sort() - .map( l => { return {'name': l, 'detect': true} } ) - self.detectorLabels = findLabel || [] - } - xhr.onerror = function (e) { - f7.dialog.alert('ALVINN has encountered an unknown server error') - return - } - - xhr.send() -} - -function remoteDetect() { - var self = this - var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect` - var xhr = new XMLHttpRequest() - xhr.open("POST", modelURL) - xhr.timeout = 10000 - xhr.ontimeout = this.remoteTimeout - xhr.setRequestHeader('Content-Type', 'application/json') - xhr.onload = function () { - self.detecting = false - if (this.status !== 200) { - console.log(xhr.response) - const errorResponse = JSON.parse(xhr.response) - f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`) - return; - } - self.resultData = JSON.parse(xhr.response) - self.uploadDirty = true - } - - var doodsData = { - "detector_name": this.detectorName, - "detect": { - "*": 1 - }, - "data": this.imageView.src.split(',')[1] - } - - xhr.send(JSON.stringify(doodsData)) -} - -function remoteTimeout () { - this.detecting = false - f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') -} - async function videoFrame (vidData) { const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) console.time('frame-process') diff --git a/src/pages/camera-mixin.js b/src/pages/camera-mixin.js index d107415..93e8e01 100644 --- a/src/pages/camera-mixin.js +++ b/src/pages/camera-mixin.js @@ -42,7 +42,7 @@ export default { this.getImage(tempCVS.toDataURL()) }, async videoFrameDetect (vidData) { - const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + const vidWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'}) vidWorker.port.onmessage = (eVid) => { self = this if (eVid.data.error) { diff --git a/src/pages/detect.vue b/src/pages/detect.vue index d901a54..ac81e18 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -177,7 +177,8 @@ videoDeviceAvailable: false, videoAvailable: false, cameraStream: null, - infoLinkPos: {} + infoLinkPos: {}, + workerScript: null } }, setup() { @@ -204,7 +205,7 @@ } this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json` this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json` - fetch(`${this.isCordova ? 'https://localhost' : '.'}/models/${this.detectorName}/classes.json`) + fetch(`${modelRoot}/models/${this.detectorName}/classes.json`) .then((mod) => { return mod.json() }) .then((classes) => { this.classesList = classes @@ -214,7 +215,7 @@ if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings) }, mounted () { - const mountWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + const mountWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'}) mountWorker.port.onmessage = (eMount) => { self = this if (eMount.data.error) { @@ -291,32 +292,43 @@ return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)` }, async setData () { - const detectWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'}) + const detectWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'}) detectWorker.port.onmessage = (eDetect) => { self = this if (eDetect.data.error) { self.detecting = false self.resultData = {} f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`) - } else { + } else if (eDetect.data.success == 'detection') { self.detecting = false self.resultData = eDetect.data.detections if (self.resultData) { self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name}) } self.uploadDirty = true + } else if (eDetect.data.success == 'model') { + this.reloadModel = false + loadSuccess(true) } } - if (this.reloadModel) { - await this.loadModel(this.modelLocation) - this.reloadModel = false - } + let loadSuccess = null + let loadFailure = null + let modelReloading = new Promise((res, rej) => { + loadSuccess = res + loadFailure = rej + if (this.reloadModel) { + detectWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation}) + } else { + loadSuccess(true) + } + }) + if (this.serverSettings && this.serverSettings.use) { this.remoteDetect() } else { - createImageBitmap(this.imageView).then(imData => { - detectWorker.port.postMessage({call: 'localDetect', image: imData}, [imData]) + Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => { + detectWorker.port.postMessage({call: 'localDetect', image: res[1]}, [res[1]]) }) } }, diff --git a/src/pages/detection-mixin.js b/src/pages/detection-mixin.js index 2bffbc5..438742c 100644 --- a/src/pages/detection-mixin.js +++ b/src/pages/detection-mixin.js @@ -1,114 +1,7 @@ -import * as tf from '@tensorflow/tfjs' import { f7 } from 'framework7-vue' -let model = null - export default { methods: { - async loadModel(weights, preload) { - if (model && model.modelURL == weights) { - return model - } else if (model) { - tf.dispose(model) - } - model = await tf.loadGraphModel(weights) - const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - /***************** - * If preloading then run model - * once on fake data to preload - * weights for a faster response - *****************/ - if (preload) { - const dummyT = tf.ones([1,modelWidth,modelHeight,3]) - model.predict(dummyT) - } - return model - }, - async localDetect(imageData) { - console.time('pre-process') - const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) - let gTense = null - const input = tf.tidy(() => { - gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0) - return tf.concat([gTense,gTense,gTense],3) - }) - tf.dispose(gTense) - console.timeEnd('pre-process') - - console.time('run prediction') - const res = model.predict(input) - const tRes = tf.transpose(res,[0,2,1]) - const rawRes = tRes.arraySync()[0] - console.timeEnd('run prediction') - - console.time('post-process') - const outputSize = res.shape[1] - let rawBoxes = [] - let rawScores = [] - - for (var i = 0; i < rawRes.length; i++) { - var getScores = rawRes[i].slice(4) - if (getScores.every( s => s < .05)) { continue } - var getBox = rawRes[i].slice(0,4) - var boxCalc = [ - (getBox[0] - (getBox[2] / 2)) / modelWidth, - (getBox[1] - (getBox[3] / 2)) / modelHeight, - (getBox[0] + (getBox[2] / 2)) / modelWidth, - (getBox[1] + (getBox[3] / 2)) / modelHeight, - ] - rawBoxes.push(boxCalc) - rawScores.push(getScores) - } - - if (rawBoxes.length > 0) { - const tBoxes = tf.tensor2d(rawBoxes) - let tScores = null - let resBoxes = null - let validBoxes = [] - let structureScores = null - let boxes_data = [] - let scores_data = [] - let classes_data = [] - for (var c = 0; c < outputSize - 4; c++) { - structureScores = rawScores.map(x => x[c]) - tScores = tf.tensor1d(structureScores) - resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05) - validBoxes = resBoxes.dataSync() - tf.dispose(resBoxes) - if (validBoxes) { - boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx))) - var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx)) - scores_data.push(...outputScores) - classes_data.push(...outputScores.fill(c)) - } - } - - validBoxes = [] - tf.dispose(tBoxes) - tf.dispose(tScores) - tf.dispose(tRes) - const valid_detections_data = classes_data.length - var output = { - detections: [] - } - for (var i =0; i < valid_detections_data; i++) { - var [dLeft, dTop, dRight, dBottom] = boxes_data[i] - output.detections.push({ - "top": dTop, - "left": dLeft, - "bottom": dBottom, - "right": dRight, - "label": this.detectorLabels[classes_data[i]].name, - "confidence": scores_data[i] * 100 - }) - } - } - tf.dispose(res) - tf.dispose(input) - console.timeEnd('post-process') - - return output || { detections: [] } - }, getRemoteLabels() { var self = this var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors` -- 2.49.1