ALVINN/ALVINN_f7
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add detection worker #187

Merged
jgeorgi merged 4 commits from xps-detect-workers into main 2024-07-25 17:56:21 +00:00
Conversation 0 Commits 4 Files Changed 4 +278 -190
4 changed files with 278 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 205f2da87d - Show all commits

74
src/js/detect-worker.js → src/assets/detect-worker.js
View File

@@ -10,22 +10,22 @@ self.onconnect = (e) => {
switch (e.data.call) {
case 'loadModel':
loadModel('.' + e.data.weights,e.data.preload).then(() => {
port.postMessage({success: true})
port.postMessage({success: 'model'})
}).catch((err) => {
port.postMessage({error: true, message: err.message})
})
break
case 'localDetect':
localDetect(e.data.image).then((dets) => {
port.postMessage({success: true, detections: dets})
port.postMessage({success: 'detection', detections: dets})
}).catch((err) => {
port.postMessage({error: true, message: err.message})
})
e.data.image.close()
break
case 'videoFrame':
videoFrame(e.data.image).then((franeDet) =>{
port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH})
videoFrame(e.data.image).then((frameDet) =>{
port.postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH})
}).catch((err) => {
port.postMessage({error: true, message: err.message})
})
@@ -149,72 +149,6 @@ async function localDetect(imageData) {
return output || { detections: [] }
}
function getRemoteLabels() {
var self = this
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
var xhr = new XMLHttpRequest()
xhr.open("GET", modelURL)
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.onload = function () {
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return
}
var detectors = JSON.parse(xhr.response).detectors
var findLabel = detectors
.find( d => { return d.name == self.detectorName } )?.labels
.filter( l => { return l != "" } ).sort()
.map( l => { return {'name': l, 'detect': true} } )
self.detectorLabels = findLabel || []
}
xhr.onerror = function (e) {
f7.dialog.alert('ALVINN has encountered an unknown server error')
return
}
xhr.send()
}
function remoteDetect() {
var self = this
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
var xhr = new XMLHttpRequest()
xhr.open("POST", modelURL)
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.onload = function () {
self.detecting = false
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
self.resultData = JSON.parse(xhr.response)
self.uploadDirty = true
}
var doodsData = {
"detector_name": this.detectorName,
"detect": {
"*": 1
},
"data": this.imageView.src.split(',')[1]
}
xhr.send(JSON.stringify(doodsData))
}
function remoteTimeout () {
this.detecting = false
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
}
async function videoFrame (vidData) {
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
console.time('frame-process')

2
src/pages/camera-mixin.js
View File

@@ -42,7 +42,7 @@ export default {
this.getImage(tempCVS.toDataURL())
},
async videoFrameDetect (vidData) {
const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
const vidWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
vidWorker.port.onmessage = (eVid) => {
self = this
if (eVid.data.error) {

30
src/pages/detect.vue
View File

@@ -177,7 +177,8 @@
videoDeviceAvailable: false,
videoAvailable: false,
cameraStream: null,
infoLinkPos: {}
infoLinkPos: {},
workerScript: null
}
},
setup() {
@@ -204,7 +205,7 @@
}
this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
fetch(`${this.isCordova ? 'https://localhost' : '.'}/models/${this.detectorName}/classes.json`)
fetch(`${modelRoot}/models/${this.detectorName}/classes.json`)
.then((mod) => { return mod.json() })
.then((classes) => {
this.classesList = classes
@@ -214,7 +215,7 @@
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
},
mounted () {
const mountWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
const mountWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
mountWorker.port.onmessage = (eMount) => {
self = this
if (eMount.data.error) {
@@ -291,32 +292,43 @@
return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
},
async setData () {
const detectWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
const detectWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
detectWorker.port.onmessage = (eDetect) => {
self = this
if (eDetect.data.error) {
self.detecting = false
self.resultData = {}
f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
} else {
} else if (eDetect.data.success == 'detection') {
self.detecting = false
self.resultData = eDetect.data.detections
if (self.resultData) {
self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name})
}
self.uploadDirty = true
} else if (eDetect.data.success == 'model') {
this.reloadModel = false
loadSuccess(true)
}
}
let loadSuccess = null
let loadFailure = null
let modelReloading = new Promise((res, rej) => {
loadSuccess = res
loadFailure = rej
if (this.reloadModel) {
await this.loadModel(this.modelLocation)
this.reloadModel = false
detectWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation})
} else {
loadSuccess(true)
}
})
if (this.serverSettings && this.serverSettings.use) {
this.remoteDetect()
} else {
createImageBitmap(this.imageView).then(imData => {
detectWorker.port.postMessage({call: 'localDetect', image: imData}, [imData])
Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
detectWorker.port.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
})
}
},

107
src/pages/detection-mixin.js
View File

@@ -1,114 +1,7 @@
import * as tf from '@tensorflow/tfjs'
import { f7 } from 'framework7-vue'
let model = null
export default {
methods: {
async loadModel(weights, preload) {
if (model && model.modelURL == weights) {
return model
} else if (model) {
tf.dispose(model)
}
model = await tf.loadGraphModel(weights)
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
/*****************
* If preloading then run model
* once on fake data to preload
* weights for a faster response
*****************/
if (preload) {
const dummyT = tf.ones([1,modelWidth,modelHeight,3])
model.predict(dummyT)
}
return model
},
async localDetect(imageData) {
console.time('pre-process')
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
let gTense = null
const input = tf.tidy(() => {
gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
return tf.concat([gTense,gTense,gTense],3)
})
tf.dispose(gTense)
console.timeEnd('pre-process')
console.time('run prediction')
const res = model.predict(input)
const tRes = tf.transpose(res,[0,2,1])
const rawRes = tRes.arraySync()[0]
console.timeEnd('run prediction')
console.time('post-process')
const outputSize = res.shape[1]
let rawBoxes = []
let rawScores = []
for (var i = 0; i < rawRes.length; i++) {
var getScores = rawRes[i].slice(4)
if (getScores.every( s => s < .05)) { continue }
var getBox = rawRes[i].slice(0,4)
var boxCalc = [
(getBox[0] - (getBox[2] / 2)) / modelWidth,
(getBox[1] - (getBox[3] / 2)) / modelHeight,
(getBox[0] + (getBox[2] / 2)) / modelWidth,
(getBox[1] + (getBox[3] / 2)) / modelHeight,
]
rawBoxes.push(boxCalc)
rawScores.push(getScores)
}
if (rawBoxes.length > 0) {
const tBoxes = tf.tensor2d(rawBoxes)
let tScores = null
let resBoxes = null
let validBoxes = []
let structureScores = null
let boxes_data = []
let scores_data = []
let classes_data = []
for (var c = 0; c < outputSize - 4; c++) {
structureScores = rawScores.map(x => x[c])
tScores = tf.tensor1d(structureScores)
resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
validBoxes = resBoxes.dataSync()
tf.dispose(resBoxes)
if (validBoxes) {
boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
scores_data.push(...outputScores)
classes_data.push(...outputScores.fill(c))
}
}
validBoxes = []
tf.dispose(tBoxes)
tf.dispose(tScores)
tf.dispose(tRes)
const valid_detections_data = classes_data.length
var output = {
detections: []
}
for (var i =0; i < valid_detections_data; i++) {
var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
output.detections.push({
"top": dTop,
"left": dLeft,
"bottom": dBottom,
"right": dRight,
"label": this.detectorLabels[classes_data[i]].name,
"confidence": scores_data[i] * 100
})
}
}
tf.dispose(res)
tf.dispose(input)
console.timeEnd('post-process')
return output || { detections: [] }
},
getRemoteLabels() {
var self = this
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`