ALVINN/ALVINN_f7
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add safari-based logic for detection calls

Browse Source 
Signed-off-by: Justin Georgi <justin.georgi@gmail.com>
This commit is contained in:
Justin Georgi
2024-08-15 15:36:38 -07:00
parent bb0b7273f9
commit df1f0f2213
4 changed files with 48 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/assets/detect-worker.js
View File

@@ -57,7 +57,7 @@ async function loadModel(weights, preload) {
}
async function localDetect(imageData) {
console.time('pre-process')
console.time('sw: pre-process')
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
let gTense = null
const input = tf.tidy(() => {
@@ -65,15 +65,15 @@ async function localDetect(imageData) {
return tf.concat([gTense,gTense,gTense],3)
})
tf.dispose(gTense)
console.timeEnd('pre-process')
console.timeEnd('sw: pre-process')
console.time('run prediction')
console.time('sw: run prediction')
const res = model.predict(input)
const tRes = tf.transpose(res,[0,2,1])
const rawRes = tRes.arraySync()[0]
console.timeEnd('run prediction')
console.timeEnd('sw: run prediction')
console.time('post-process')
console.time('sw: post-process')
const outputSize = res.shape[1]
let rawBoxes = []
let rawScores = []
@@ -138,14 +138,14 @@ async function localDetect(imageData) {
}
tf.dispose(res)
tf.dispose(input)
console.timeEnd('post-process')
console.timeEnd('sw: post-process')
return output || { detections: [] }
}
async function videoFrame (vidData) {
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
console.time('frame-process')
console.time('sw: frame-process')
let rawCoords = []
try {
const input = tf.tidy(() => {
@@ -171,6 +171,6 @@ async function videoFrame (vidData) {
} catch (e) {
console.log(e)
}
console.timeEnd('frame-process')
console.timeEnd('sw: frame-process')
return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
}

2
src/pages/camera-mixin.js
View File

@@ -41,7 +41,7 @@ export default {
tempCtx.drawImage(vidViewer, 0, 0)
this.getImage(tempCVS.toDataURL())
},
async videoFrameDetect (vidData) {
async videoFrameDetectWorker (vidData) {
const startDetection = () => {
createImageBitmap(vidData).then(imVideoFrame => {
this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])

43
src/pages/detect.vue
View File

@@ -337,22 +337,39 @@
let loadSuccess = null
let loadFailure = null
let modelReloading = new Promise((res, rej) => {
loadSuccess = res
loadFailure = rej
if (this.reloadModel) {
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
} else {
loadSuccess()
}
})
let modelReloading = null
if (this.isSafari && this.reloadModel) {
await this.loadModel(this.modelLocation)
this.reloadModel = false
} else {
modelReloading = new Promise((res, rej) => {
loadSuccess = res
loadFailure = rej
if (this.reloadModel) {
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
} else {
loadSuccess()
}
})
}
if (this.serverSettings && this.serverSettings.use) {
this.remoteDetect()
} else {
} else if (!this.isSafari) {
Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
})
} else {
this.localDetect(this.imageView).then(dets => {
this.detecting = false
this.resultData = dets
this.uploadDirty = true
}).catch((e) => {
console.log(e.message)
this.detecting = false
this.resultData = {}
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
})
}
},
selectAll (ev) {
@@ -368,7 +385,7 @@
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
return
}
if (mode == "camera") {
if (mode == "camera" && !this.otherSettings.disableVideo) {
this.videoAvailable = await this.openCamera(this.$refs.image_container)
if (this.videoAvailable) {
this.selectedChip = -1
@@ -380,8 +397,10 @@
var vidElement = this.$refs.vid_viewer
vidElement.width = trackDetails.width
vidElement.height = trackDetails.height
if (!this.otherSettings.disableVideo) {
if (this.isSafari) {
this.videoFrameDetect(vidElement)
} else {
this.videoFrameDetectWorker(vidElement)
}
return
}

16
src/pages/detection-mixin.js
View File

@@ -25,7 +25,7 @@ export default {
return model
},
async localDetect(imageData) {
console.time('pre-process')
console.time('mx: pre-process')
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
let gTense = null
const input = tf.tidy(() => {
@@ -33,15 +33,15 @@ export default {
return tf.concat([gTense,gTense,gTense],3)
})
tf.dispose(gTense)
console.timeEnd('pre-process')
console.timeEnd('mx: pre-process')
console.time('run prediction')
console.time('mx: run prediction')
const res = model.predict(input)
const tRes = tf.transpose(res,[0,2,1])
const rawRes = tRes.arraySync()[0]
console.timeEnd('run prediction')
console.timeEnd('mx: run prediction')
console.time('post-process')
console.time('mx: post-process')
const outputSize = res.shape[1]
let rawBoxes = []
let rawScores = []
@@ -105,7 +105,7 @@ export default {
}
tf.dispose(res)
tf.dispose(input)
console.timeEnd('post-process')
console.timeEnd('mx: post-process')
return output || { detections: [] }
},
@@ -194,7 +194,7 @@ export default {
imgHeight = imCanvas.height
}
while (this.videoAvailable) {
console.time('frame-process')
console.time('mx: frame-process')
try {
const input = tf.tidy(() => {
return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
@@ -228,7 +228,7 @@ export default {
} catch (e) {
console.log(e)
}
console.timeEnd('frame-process')
console.timeEnd('mx: frame-process')
await tf.nextFrame();
}
}