ALVINN/ALVINN_f7
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add real-time detection to camera stream (#143)

Browse Source 
Closes: #30

When the camera is being used to find an image to capture, the region mini model now runs in real time to give an estimate of where there are identifiable structures.

Reviewed-on: #143
This commit is contained in:
Justin Georgi
2024-03-24 08:51:08 -07:00
parent f09180875a
commit 79316bb83b
21 changed files with 417 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/pages/detect.vue
View File

@@ -6,12 +6,12 @@
</f7-navbar>
<f7-block class="detect-grid">
<div class="image-container" ref="image_container">
<canvas id="im-draw" ref="image_cvs" @click="structureClick" :style="`display: ${imageLoaded ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat`" />
<SvgIcon v-if="!imageView && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)" @click="selectImage" />
<div class="vid-container" v-if="videoAvailable" style="width: 100%; height: 100%">
<div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
<video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
<f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%);" fill large>Capture</f7-button>
<f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
</div>
<canvas id="im-draw" ref="image_cvs" @click="structureClick" :style="`display: ${(imageLoaded || videoAvailable) ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat; z-index: 2;`" />
</div>
<div class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
<f7-chip v-for="result in showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted })"
@@ -56,6 +56,7 @@
</f7-button>
</f7-segmented>
<input type="file" ref="image_chooser" @change="getImage()" accept="image/*" style="display: none;"/>
<img src="../assets/target.svg" ref="target_image" style="display: none;" />
</f7-block>
<f7-panel :id="detectorName + '-settings'" right cover :backdrop="false" :container-el="`#${detectorName}-detect-page`">
@@ -147,7 +148,9 @@
uploadUid: null,
uploadDirty: false,
modelLocation: '',
miniLocation: '',
modelLoading: true,
reloadModel: false,
videoDeviceAvailable: false,
videoAvailable: false,
cameraStream: null
@@ -163,17 +166,23 @@
case 'thorax':
this.activeRegion = 0
this.detectorName = 'thorax'
//this.classesList = thoraxClasses
/* VITE setting */
this.modelLocation = `../models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.miniLocation = `../models/thorax-mini/model.json`
/* PWA Build setting */
//this.modelLocation = `./models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.modelLocationCordova = `https://localhost/models/thorax${this.otherSettings.mini ? '-mini' : ''}/model.json`
break;
case 'abdomen':
this.activeRegion = 1
this.detectorName = 'combined'
break;
this.detectorName = 'abdomen'
/* VITE setting */
this.modelLocation = `../models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.miniLocation = `../models/abdomen-mini/model.json`
/* PWA Build setting */
//this.modelLocation = `./models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
this.modelLocationCordova = `https://localhost/models/abdomen${this.otherSettings.mini ? '-mini' : ''}/model.json`
break;
case 'limbs':
this.activeRegion = 2
this.detectorName = 'defaultNew'
@@ -195,7 +204,7 @@
this.modelLoading = false
} else {
this.modelLoading = true
this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation).then(() => {
this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation, true).then(() => {
this.modelLoading = false
}).catch((e) => {
console.log(e.message)
@@ -253,7 +262,11 @@
chipGradient (confVal) {
return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
},
setData () {
async setData () {
if (this.reloadModel) {
await this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation)
this.reloadModel = false
}
if (this.serverSettings && this.serverSettings.use) {
this.remoteDetect()
} else {
@@ -284,7 +297,20 @@
}
if (mode == "camera") {
this.videoAvailable = await this.openCamera(this.$refs.image_container)
if (this.videoAvailable) { return }
if (this.videoAvailable) {
this.imageLoaded = false
this.imageView = null
this.$refs.image_cvs.style['background-image'] = 'none'
this.resultData = {}
var trackDetails = this.cameraStream.getVideoTracks()[0].getSettings()
var vidElement = this.$refs.vid_viewer
vidElement.width = trackDetails.width
vidElement.height = trackDetails.height
if (!this.otherSettings.disableVideo) {
this.videoFrameDetect(vidElement)
}
return
}
}
if (mode == 'sample') {
f7.dialog.create({
@@ -350,6 +376,7 @@
if (this.videoAvailable) {
this.closeCamera()
this.detecting = true
this.reloadModel = true
resolve(searchImage)
} else if (this.isCordova && this.imageLoadMode == "camera") {
this.detecting = true
@@ -421,7 +448,7 @@
this.selectChip(findBox >= 0 ? this.resultData.detections[findBox].resultIndex : this.selectedChip)
},
box2cvs(boxInput) {
if (!boxInput) return []
if (!boxInput || boxInput.length == 0) return []
const boxList = boxInput.length ? boxInput : [boxInput]
const [imCanvas, imageCtx] = this.resetView()
var imgWidth