Add real-time detection to camera stream #143

Merged
jgeorgi merged 5 commits from xps-video-stream into main 2024-03-24 15:51:10 +00:00
20 changed files with 401 additions and 18 deletions
Showing only changes of commit 2046fca96a - Show all commits

View File

@@ -148,7 +148,9 @@
uploadUid: null, uploadUid: null,
uploadDirty: false, uploadDirty: false,
modelLocation: '', modelLocation: '',
miniLocation: '',
modelLoading: true, modelLoading: true,
reloadModel: false,
videoDeviceAvailable: false, videoDeviceAvailable: false,
videoAvailable: false, videoAvailable: false,
cameraStream: null cameraStream: null
@@ -166,6 +168,7 @@
this.detectorName = 'thorax' this.detectorName = 'thorax'
/* VITE setting */ /* VITE setting */
this.modelLocation = `../models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json` this.modelLocation = `../models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.miniLocation = `../models/thorax-mini/model.json`
/* PWA Build setting */ /* PWA Build setting */
//this.modelLocation = `./models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json` //this.modelLocation = `./models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.modelLocationCordova = `https://localhost/models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json` this.modelLocationCordova = `https://localhost/models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
@@ -175,6 +178,7 @@
this.detectorName = 'abdomen' this.detectorName = 'abdomen'
/* VITE setting */ /* VITE setting */
this.modelLocation = `../models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json` this.modelLocation = `../models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.miniLocation = `../models/abdomen-mini/model.json`
/* PWA Build setting */ /* PWA Build setting */
//this.modelLocation = `./models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json` //this.modelLocation = `./models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.modelLocationCordova = `https://localhost/models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json` this.modelLocationCordova = `https://localhost/models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
@@ -200,7 +204,7 @@
this.modelLoading = false this.modelLoading = false
} else { } else {
this.modelLoading = true this.modelLoading = true
this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation).then(() => { this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation, true).then(() => {
this.modelLoading = false this.modelLoading = false
}).catch((e) => { }).catch((e) => {
console.log(e.message) console.log(e.message)
@@ -258,7 +262,11 @@
chipGradient (confVal) { chipGradient (confVal) {
return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)` return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
}, },
setData () { async setData () {
if (this.reloadModel) {
await this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation)
this.reloadModel = false
}
if (this.serverSettings && this.serverSettings.use) { if (this.serverSettings && this.serverSettings.use) {
this.remoteDetect() this.remoteDetect()
} else { } else {
@@ -294,7 +302,9 @@
var vidElement = this.$refs.vid_viewer var vidElement = this.$refs.vid_viewer
vidElement.width = trackDetails.width vidElement.width = trackDetails.width
vidElement.height = trackDetails.height vidElement.height = trackDetails.height
this.videoFrameDetect(vidElement) if (!this.otherSettings.disableVideo) {
this.videoFrameDetect(vidElement)
}
return return
} }
} }
@@ -362,6 +372,7 @@
if (this.videoAvailable) { if (this.videoAvailable) {
this.closeCamera() this.closeCamera()
this.detecting = true this.detecting = true
this.reloadModel = true
resolve(searchImage) resolve(searchImage)
} else if (this.isCordova && this.imageLoadMode == "camera") { } else if (this.isCordova && this.imageLoadMode == "camera") {
this.detecting = true this.detecting = true

View File

@@ -5,14 +5,23 @@ var model = null
export default { export default {
methods: { methods: {
async loadModel(weights) { async loadModel(weights, preload) {
if (model && model.modelURL == weights) {
return model
} else if (model) {
model.dispose()
}
model = await tf.loadGraphModel(weights) model = await tf.loadGraphModel(weights)
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
const dummyT = tf.ones([1,modelWidth,modelHeight,3])
/***************** /*****************
* Run model once to preload weights for better response time * If preloading then run model
* once on fake data to preload
* weights for a faster response
*****************/ *****************/
model.predict(dummyT) if (preload) {
const dummyT = tf.ones([1,modelWidth,modelHeight,3])
model.predict(dummyT)
}
return model return model
}, },
async localDetect(imageData) { async localDetect(imageData) {
@@ -155,6 +164,7 @@ export default {
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.') f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
}, },
async videoFrameDetect (vidData) { async videoFrameDetect (vidData) {
await this.loadModel(this.miniLocation)
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3) const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
const imCanvas = this.$refs.image_cvs const imCanvas = this.$refs.image_cvs
const imageCtx = imCanvas.getContext("2d") const imageCtx = imCanvas.getContext("2d")

View File

@@ -27,6 +27,10 @@
<span style="margin-left: 16px;">Enable demo mode</span> <span style="margin-left: 16px;">Enable demo mode</span>
<f7-toggle v-model:checked="otherSettings.demo" style="margin-right: 16px;" @change="setDirty()" /> <f7-toggle v-model:checked="otherSettings.demo" style="margin-right: 16px;" @change="setDirty()" />
</div> </div>
<div style="display:flex; justify-content:space-between; width: 100%; margin-bottom: 10px;">
<span style="margin-left: 16px;">Disable video estimates<f7-icon size="16" style="padding-left: 5px;" f7="question_diamond_fill" tooltip="faster: recommended for slower devices" /></span>
<f7-toggle v-model:checked="otherSettings.disableVideo" style="margin-right: 16px;" />
</div>
<div style="display:flex; justify-content:space-between; width: 100%"> <div style="display:flex; justify-content:space-between; width: 100%">
<span style="margin-left: 16px;">Use external server</span> <span style="margin-left: 16px;">Use external server</span>
<f7-toggle v-model:checked="serverSettings.use" style="margin-right: 16px;" @change="setDirty()" /> <f7-toggle v-model:checked="serverSettings.use" style="margin-right: 16px;" @change="setDirty()" />