Justin Georgi
a98577e206
All checks were successful
Build Dev PWA / Build-PWA (push) Successful in 37s
Signed-off-by: Justin Georgi <justin.georgi@gmail.com>
236 lines
8.2 KiB
JavaScript
236 lines
8.2 KiB
JavaScript
import * as tf from '@tensorflow/tfjs'
|
|
import { f7 } from 'framework7-vue'
|
|
|
|
let model = null
|
|
|
|
export default {
|
|
methods: {
|
|
async loadModel(weights, preload) {
|
|
if (model && model.modelURL == weights) {
|
|
return model
|
|
} else if (model) {
|
|
tf.dispose(model)
|
|
}
|
|
model = await tf.loadGraphModel(weights)
|
|
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
/*****************
|
|
* If preloading then run model
|
|
* once on fake data to preload
|
|
* weights for a faster response
|
|
*****************/
|
|
if (preload) {
|
|
const dummyT = tf.ones([1,modelWidth,modelHeight,3])
|
|
model.predict(dummyT)
|
|
}
|
|
return model
|
|
},
|
|
async localDetect(imageData) {
|
|
console.time('mx: pre-process')
|
|
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
let gTense = null
|
|
const input = tf.tidy(() => {
|
|
gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
|
|
return tf.concat([gTense,gTense,gTense],3)
|
|
})
|
|
tf.dispose(gTense)
|
|
console.timeEnd('mx: pre-process')
|
|
|
|
console.time('mx: run prediction')
|
|
const res = model.predict(input)
|
|
const tRes = tf.transpose(res,[0,2,1])
|
|
const rawRes = tRes.arraySync()[0]
|
|
console.timeEnd('mx: run prediction')
|
|
|
|
console.time('mx: post-process')
|
|
const outputSize = res.shape[1]
|
|
let rawBoxes = []
|
|
let rawScores = []
|
|
|
|
for (var i = 0; i < rawRes.length; i++) {
|
|
var getScores = rawRes[i].slice(4)
|
|
if (getScores.every( s => s < .05)) { continue }
|
|
var getBox = rawRes[i].slice(0,4)
|
|
var boxCalc = [
|
|
(getBox[0] - (getBox[2] / 2)) / modelWidth,
|
|
(getBox[1] - (getBox[3] / 2)) / modelHeight,
|
|
(getBox[0] + (getBox[2] / 2)) / modelWidth,
|
|
(getBox[1] + (getBox[3] / 2)) / modelHeight,
|
|
]
|
|
rawBoxes.push(boxCalc)
|
|
rawScores.push(getScores)
|
|
}
|
|
|
|
if (rawBoxes.length > 0) {
|
|
const tBoxes = tf.tensor2d(rawBoxes)
|
|
let tScores = null
|
|
let resBoxes = null
|
|
let validBoxes = []
|
|
let structureScores = null
|
|
let boxes_data = []
|
|
let scores_data = []
|
|
let classes_data = []
|
|
for (var c = 0; c < outputSize - 4; c++) {
|
|
structureScores = rawScores.map(x => x[c])
|
|
tScores = tf.tensor1d(structureScores)
|
|
resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
|
|
validBoxes = resBoxes.dataSync()
|
|
tf.dispose(resBoxes)
|
|
if (validBoxes) {
|
|
boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
|
|
var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
|
|
scores_data.push(...outputScores)
|
|
classes_data.push(...outputScores.fill(c))
|
|
}
|
|
}
|
|
|
|
validBoxes = []
|
|
tf.dispose(tBoxes)
|
|
tf.dispose(tScores)
|
|
tf.dispose(tRes)
|
|
const valid_detections_data = classes_data.length
|
|
var output = {
|
|
detections: []
|
|
}
|
|
for (var i =0; i < valid_detections_data; i++) {
|
|
var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
|
|
output.detections.push({
|
|
"top": dTop,
|
|
"left": dLeft,
|
|
"bottom": dBottom,
|
|
"right": dRight,
|
|
"label": this.detectorLabels[classes_data[i]].name,
|
|
"confidence": scores_data[i] * 100
|
|
})
|
|
}
|
|
}
|
|
tf.dispose(res)
|
|
tf.dispose(input)
|
|
console.timeEnd('mx: post-process')
|
|
|
|
return output || { detections: [] }
|
|
},
|
|
getRemoteLabels() {
|
|
var self = this
|
|
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
|
|
var xhr = new XMLHttpRequest()
|
|
xhr.open("GET", modelURL)
|
|
xhr.setRequestHeader('Content-Type', 'application/json')
|
|
xhr.timeout = 10000
|
|
xhr.ontimeout = this.remoteTimeout
|
|
xhr.onload = function () {
|
|
if (this.status !== 200) {
|
|
console.log(xhr.response)
|
|
const errorResponse = JSON.parse(xhr.response)
|
|
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
|
|
return
|
|
}
|
|
var detectors = JSON.parse(xhr.response).detectors
|
|
var findLabel = detectors
|
|
.find( d => { return d.name == self.detectorName } )?.labels
|
|
.filter( l => { return l != "" } ).sort()
|
|
.map( l => { return {'name': l, 'detect': true} } )
|
|
self.detectorLabels = findLabel || []
|
|
}
|
|
xhr.onerror = function (e) {
|
|
f7.dialog.alert('ALVINN has encountered an unknown server error')
|
|
return
|
|
}
|
|
|
|
xhr.send()
|
|
},
|
|
remoteDetect() {
|
|
var self = this
|
|
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
|
|
var xhr = new XMLHttpRequest()
|
|
xhr.open("POST", modelURL)
|
|
xhr.timeout = 10000
|
|
xhr.ontimeout = this.remoteTimeout
|
|
xhr.setRequestHeader('Content-Type', 'application/json')
|
|
xhr.onload = function () {
|
|
self.detecting = false
|
|
if (this.status !== 200) {
|
|
console.log(xhr.response)
|
|
const errorResponse = JSON.parse(xhr.response)
|
|
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
|
|
return;
|
|
}
|
|
self.resultData = JSON.parse(xhr.response)
|
|
self.uploadDirty = true
|
|
}
|
|
|
|
var doodsData = {
|
|
"detector_name": this.detectorName,
|
|
"detect": {
|
|
"*": 1
|
|
},
|
|
"data": this.imageView.src.split(',')[1]
|
|
}
|
|
|
|
xhr.send(JSON.stringify(doodsData))
|
|
},
|
|
remoteTimeout () {
|
|
this.detecting = false
|
|
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
|
|
},
|
|
async videoFrameDetect (vidData, miniModel) {
|
|
await this.loadModel(miniModel)
|
|
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
const imCanvas = this.$refs.image_cvs
|
|
const imageCtx = imCanvas.getContext("2d")
|
|
const target = this.$refs.target_image
|
|
await tf.nextFrame();
|
|
imCanvas.width = imCanvas.clientWidth
|
|
imCanvas.height = imCanvas.clientHeight
|
|
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
|
var imgWidth
|
|
var imgHeight
|
|
const imgAspect = vidData.width / vidData.height
|
|
const rendAspect = imCanvas.width / imCanvas.height
|
|
if (imgAspect >= rendAspect) {
|
|
imgWidth = imCanvas.width
|
|
imgHeight = imCanvas.width / imgAspect
|
|
} else {
|
|
imgWidth = imCanvas.height * imgAspect
|
|
imgHeight = imCanvas.height
|
|
}
|
|
while (this.videoAvailable) {
|
|
console.time('mx: frame-process')
|
|
try {
|
|
const input = tf.tidy(() => {
|
|
return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
|
|
})
|
|
const res = model.predict(input)
|
|
const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
|
|
|
|
let rawCoords = []
|
|
if (rawRes) {
|
|
for (var i = 0; i < rawRes.length; i++) {
|
|
let getScores = rawRes[i].slice(4)
|
|
if (getScores.some( s => s > .5)) {
|
|
let foundTarget = rawRes[i].slice(0,2)
|
|
foundTarget.push(Math.max(...getScores))
|
|
rawCoords.push(foundTarget)
|
|
}
|
|
}
|
|
|
|
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
|
for (var coord of rawCoords) {
|
|
console.log(`x: ${coord[0]}, y: ${coord[1]}`)
|
|
let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
|
|
let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
|
|
imageCtx.globalAlpha = coord[2]
|
|
imageCtx.drawImage(target, pointX, pointY, 20, 20)
|
|
}
|
|
}
|
|
tf.dispose(input)
|
|
tf.dispose(res)
|
|
tf.dispose(rawRes)
|
|
} catch (e) {
|
|
console.log(e)
|
|
}
|
|
console.timeEnd('mx: frame-process')
|
|
await tf.nextFrame();
|
|
}
|
|
}
|
|
}
|
|
} |