ALVINN/ALVINN_f7
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add safari-based logic for detection calls

Browse Source 
Signed-off-by: Justin Georgi <justin.georgi@gmail.com>
This commit is contained in:
Justin Georgi
2024-08-15 15:36:38 -07:00
parent bb0b7273f9
commit df1f0f2213
4 changed files with 48 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/assets/detect-worker.js
View File

@@ -57,7 +57,7 @@ async function loadModel(weights, preload) {
} }
async function localDetect(imageData) { async function localDetect(imageData) {
console.time('pre-process') console.time('sw: pre-process')
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
let gTense = null let gTense = null
const input = tf.tidy(() => { const input = tf.tidy(() => {
@@ -65,15 +65,15 @@ async function localDetect(imageData) {
return tf.concat([gTense,gTense,gTense],3) return tf.concat([gTense,gTense,gTense],3)
}) })
tf.dispose(gTense) tf.dispose(gTense)
console.timeEnd('pre-process') console.timeEnd('sw: pre-process')
console.time('run prediction') console.time('sw: run prediction')
const res = model.predict(input) const res = model.predict(input)
const tRes = tf.transpose(res,[0,2,1]) const tRes = tf.transpose(res,[0,2,1])
const rawRes = tRes.arraySync()[0] const rawRes = tRes.arraySync()[0]
console.timeEnd('run prediction') console.timeEnd('sw: run prediction')
console.time('post-process') console.time('sw: post-process')
const outputSize = res.shape[1] const outputSize = res.shape[1]
let rawBoxes = [] let rawBoxes = []
let rawScores = [] let rawScores = []
@@ -138,14 +138,14 @@ async function localDetect(imageData) {
} }
tf.dispose(res) tf.dispose(res)
tf.dispose(input) tf.dispose(input)
console.timeEnd('post-process') console.timeEnd('sw: post-process')
return output || { detections: [] } return output || { detections: [] }
} }
async function videoFrame (vidData) { async function videoFrame (vidData) {
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
console.time('frame-process') console.time('sw: frame-process')
let rawCoords = [] let rawCoords = []
try { try {
const input = tf.tidy(() => { const input = tf.tidy(() => {
@@ -171,6 +171,6 @@ async function videoFrame (vidData) {
} catch (e) { } catch (e) {
console.log(e) console.log(e)
} }
console.timeEnd('frame-process') console.timeEnd('sw: frame-process')
return {cds: rawCoords, mW: modelWidth, mH: modelHeight} return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
} }

2
src/pages/camera-mixin.js
View File

@@ -41,7 +41,7 @@ export default {
tempCtx.drawImage(vidViewer, 0, 0) tempCtx.drawImage(vidViewer, 0, 0)
this.getImage(tempCVS.toDataURL()) this.getImage(tempCVS.toDataURL())
}, },
async videoFrameDetect (vidData) { async videoFrameDetectWorker (vidData) {
const startDetection = () => { const startDetection = () => {
createImageBitmap(vidData).then(imVideoFrame => { createImageBitmap(vidData).then(imVideoFrame => {
this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame]) this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])

43
src/pages/detect.vue
View File

@@ -337,22 +337,39 @@
let loadSuccess = null let loadSuccess = null
let loadFailure = null let loadFailure = null
let modelReloading = new Promise((res, rej) => { let modelReloading = null
loadSuccess = res if (this.isSafari && this.reloadModel) {
loadFailure = rej await this.loadModel(this.modelLocation)
if (this.reloadModel) { this.reloadModel = false
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation}) } else {
} else { modelReloading = new Promise((res, rej) => {
loadSuccess() loadSuccess = res
} loadFailure = rej
}) if (this.reloadModel) {
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
} else {
loadSuccess()
}
})
}
if (this.serverSettings && this.serverSettings.use) { if (this.serverSettings && this.serverSettings.use) {
this.remoteDetect() this.remoteDetect()
} else { } else if (!this.isSafari) {
Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => { Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]]) this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
}) })
} else {
this.localDetect(this.imageView).then(dets => {
this.detecting = false
this.resultData = dets
this.uploadDirty = true
}).catch((e) => {
console.log(e.message)
this.detecting = false
this.resultData = {}
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
})
} }
}, },
selectAll (ev) { selectAll (ev) {
@@ -368,7 +385,7 @@
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true }); navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
return return
} }
if (mode == "camera") { if (mode == "camera" && !this.otherSettings.disableVideo) {
this.videoAvailable = await this.openCamera(this.$refs.image_container) this.videoAvailable = await this.openCamera(this.$refs.image_container)
if (this.videoAvailable) { if (this.videoAvailable) {
this.selectedChip = -1 this.selectedChip = -1
@@ -380,8 +397,10 @@
var vidElement = this.$refs.vid_viewer var vidElement = this.$refs.vid_viewer
vidElement.width = trackDetails.width vidElement.width = trackDetails.width
vidElement.height = trackDetails.height vidElement.height = trackDetails.height
if (!this.otherSettings.disableVideo) { if (this.isSafari) {
this.videoFrameDetect(vidElement) this.videoFrameDetect(vidElement)
} else {
this.videoFrameDetectWorker(vidElement)
} }
return return
} }

16
src/pages/detection-mixin.js
View File

@@ -25,7 +25,7 @@ export default {
return model return model
}, },
async localDetect(imageData) { async localDetect(imageData) {
console.time('pre-process') console.time('mx: pre-process')
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
let gTense = null let gTense = null
const input = tf.tidy(() => { const input = tf.tidy(() => {
@@ -33,15 +33,15 @@ export default {
return tf.concat([gTense,gTense,gTense],3) return tf.concat([gTense,gTense,gTense],3)
}) })
tf.dispose(gTense) tf.dispose(gTense)
console.timeEnd('pre-process') console.timeEnd('mx: pre-process')
console.time('run prediction') console.time('mx: run prediction')
const res = model.predict(input) const res = model.predict(input)
const tRes = tf.transpose(res,[0,2,1]) const tRes = tf.transpose(res,[0,2,1])
const rawRes = tRes.arraySync()[0] const rawRes = tRes.arraySync()[0]
console.timeEnd('run prediction') console.timeEnd('mx: run prediction')
console.time('post-process') console.time('mx: post-process')
const outputSize = res.shape[1] const outputSize = res.shape[1]
let rawBoxes = [] let rawBoxes = []
let rawScores = [] let rawScores = []
@@ -105,7 +105,7 @@ export default {
} }
tf.dispose(res) tf.dispose(res)
tf.dispose(input) tf.dispose(input)
console.timeEnd('post-process') console.timeEnd('mx: post-process')
return output || { detections: [] } return output || { detections: [] }
}, },
@@ -194,7 +194,7 @@ export default {
imgHeight = imCanvas.height imgHeight = imCanvas.height
} }
while (this.videoAvailable) { while (this.videoAvailable) {
console.time('frame-process') console.time('mx: frame-process')
try { try {
const input = tf.tidy(() => { const input = tf.tidy(() => {
return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0) return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
@@ -228,7 +228,7 @@ export default {
} catch (e) { } catch (e) {
console.log(e) console.log(e)
} }
console.timeEnd('frame-process') console.timeEnd('mx: frame-process')
await tf.nextFrame(); await tf.nextFrame();
} }
} }