Split detect.vue (#122)

Closes: #112

New mixins for camera and detection (remote and local) and new css for detection page.

Reviewed-on: #122
This commit is contained in:
2024-03-06 14:36:27 -07:00
parent b00b5cf492
commit 1f25a75cec
4 changed files with 312 additions and 286 deletions

183
src/css/detect.css Normal file
View File

@@ -0,0 +1,183 @@
/*Styles for the structure detection page*/
/*Basic style*/
.detect-grid {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr 56px auto min-content;
grid-template-areas:
"image-view"
"result-view"
"detect-settings"
"menu-view";
justify-items: center;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
}
.image-container {
grid-area: image-view;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
position: relative;
display: flex;
align-self: stretch;
}
.popover-button-menu {
max-width: 90vw;
max-height: 90vh;
width: auto;
}
.segment-button-menu {
flex-wrap: nowrap;
flex-direction: column;
max-height: 100%;
min-height: 0px;
}
.chip-media {
background-color: var(--chip-media-background) !important;
}
.chip-results {
display: flex;
flex-wrap: wrap;
gap: 5px;
padding: 10px;
--f7-chip-border-radius: 16px;
--f7-chip-media-size: 32px;
--f7-chip-font-weight: normal;
overflow-y: auto;
max-height: 100%;
}
.chip-results .chip {
padding-left: 8px;
}
.selected-chip {
font-weight: 500;
box-shadow: 4px 4px 1px var(--avn-theme-color);
transform: translate(-2px, -2px);
}
.detect-inputs {
display: flex;
align-items: center;
margin: 5px;
width: 100%;
max-width: 400px;
min-width: 192px;
}
.level-slide-vert {
display: none;
}
.image-menu {
grid-area: menu-view;
margin: 5px;
max-width: 400px;
min-width: 192px;
}
.image-menu .button {
aspect-ratio: 1;
height: auto;
padding: 5px;
flex: 1 1 0%;
}
.image-menu > .button > svg {
aspect-ratio: 1;
height: auto;
width: 100%;
}
.segment-button-menu .button {
padding: 8px;
aspect-ratio: 1;
width: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
/*Additional styles for small format landscape orientation*/
@media (max-height: 450px) and (orientation: landscape) {
.detect-grid {
grid-template-columns: minmax(0,1fr) minmax(56px,max-content) auto auto;
grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
grid-template-areas:
"image-view result-view detect-settings menu-view";
justify-items: stretch;
align-items: stretch;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
position: relative;
}
.chip-results {
flex-direction: column;
max-height: 100%;
justify-self: start;
flex-wrap: nowrap;
}
.detect-inputs {
flex-direction: column;
min-width: 0;
max-width: 72px;
}
.level-slide-horz {
display: none;
}
.level-slide-vert {
display: block;
}
.image-container {
flex-direction: column;
}
.image-menu {
flex-direction: column;
aspect-ratio: .25;
width: auto;
min-width: 0;
height: 100%;
}
.image-menu .button {
aspect-ratio: 1;
width: auto;
height: 100%;
flex: 1 1 0%;
border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
border-bottom-left-radius: 0px !important;
}
.segment-button-menu {
flex-direction: row;
max-height: 100%;
min-height: 0px;
}
.segment-button-menu .button {
height: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
.button > svg {
width: 100%;
height: auto;
}
}

45
src/pages/camera-mixin.js Normal file
View File

@@ -0,0 +1,45 @@
export default {
methods: {
async openCamera() {
var cameraLoaded = false
const devicesList = await navigator.mediaDevices.enumerateDevices()
this.videoDeviceAvailable = devicesList.some( d => d.kind == "videoinput")
if (this.videoDeviceAvailable) {
navigator.mediaDevices.getUserMedia({video: true})
var vidConstraint = {
video: {
width: {
ideal: 1920
},
height: {
ideal: 1080
},
facingMode: 'environment'
},
audio: false
}
const stream = await navigator.mediaDevices.getUserMedia(vidConstraint);
cameraLoaded = true
this.cameraStream = stream
}
return cameraLoaded
},
closeCamera () {
this.cameraStream.getTracks().forEach( t => t.stop())
this.videoAvailable = false
},
captureVidFrame() {
const vidViewer = this.$refs.vid_viewer
vidViewer.pause()
let tempCVS = document.createElement('canvas')
tempCVS.height = vidViewer.videoHeight || parseInt(vidViewer.style.height)
tempCVS.width = vidViewer.videoWidth || parseInt(vidViewer.style.width)
const tempCtx = tempCVS.getContext('2d')
tempCtx.drawImage(vidViewer, 0, 0)
this.getImage(tempCVS.toDataURL())
},
async videoStream() {
//TODO
}
}
}

View File

@@ -99,189 +99,7 @@
</f7-page>
</template>
<style>
.detect-grid {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr 56px auto min-content;
grid-template-areas:
"image-view"
"result-view"
"detect-settings"
"menu-view";
justify-items: center;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
}
.image-container {
grid-area: image-view;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
position: relative;
display: flex;
align-self: stretch;
}
.popover-button-menu {
max-width: 90vw;
max-height: 90vh;
width: auto;
}
.segment-button-menu {
flex-wrap: nowrap;
flex-direction: column;
max-height: 100%;
min-height: 0px;
}
.chip-media {
background-color: var(--chip-media-background) !important;
}
.chip-results {
display: flex;
flex-wrap: wrap;
gap: 5px;
padding: 10px;
--f7-chip-border-radius: 16px;
--f7-chip-media-size: 32px;
--f7-chip-font-weight: normal;
overflow-y: auto;
max-height: 100%;
}
.chip-results .chip {
padding-left: 8px;
}
.selected-chip {
font-weight: 500;
box-shadow: 4px 4px 1px var(--avn-theme-color);
transform: translate(-2px, -2px);
}
.detect-inputs {
display: flex;
align-items: center;
margin: 5px;
width: 100%;
max-width: 400px;
min-width: 192px;
}
.level-slide-vert {
display: none;
}
.image-menu {
grid-area: menu-view;
margin: 5px;
/*width: 90%;*/
max-width: 400px;
min-width: 192px;
}
.image-menu .button {
aspect-ratio: 1;
height: auto;
padding: 5px;
flex: 1 1 0%;
}
.image-menu > .button > svg {
aspect-ratio: 1;
height: auto;
width: 100%;
}
.segment-button-menu .button {
padding: 8px;
aspect-ratio: 1;
width: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
@media (max-height: 450px) and (orientation: landscape) {
.detect-grid {
grid-template-columns: minmax(0,1fr) minmax(56px,max-content) auto auto;
grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
grid-template-areas:
"image-view result-view detect-settings menu-view";
justify-items: stretch;
align-items: stretch;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
position: relative;
}
.chip-results {
flex-direction: column;
max-height: 100%;
justify-self: start;
flex-wrap: nowrap;
}
.detect-inputs {
flex-direction: column;
min-width: 0;
max-width: 72px;
}
.level-slide-horz {
display: none;
}
.level-slide-vert {
display: block;
}
.image-container {
flex-direction: column;
}
.image-menu {
flex-direction: column;
aspect-ratio: .25;
width: auto;
min-width: 0;
height: 100%;
}
.image-menu .button {
aspect-ratio: 1;
width: auto;
height: 100%;
flex: 1 1 0%;
border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
border-bottom-left-radius: 0px !important;
}
.segment-button-menu {
flex-direction: row;
max-height: 100%;
min-height: 0px;
}
.segment-button-menu .button {
height: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
.button > svg {
width: 100%;
height: auto;
}
}
</style>
<style src="../css/detect.css" />
<script>
import { f7 } from 'framework7-vue'
@@ -291,12 +109,13 @@
import SvgIcon from '../components/svg-icon.vue'
import submitMixin from './submit-mixin'
import detectMixin from './local-detect'
import detectionMixin from './detection-mixin'
import cameraMixin from './camera-mixin'
import thoraxClasses from '../models/thorax/classes.json'
export default {
mixins: [submitMixin, detectMixin],
mixins: [submitMixin, detectionMixin, cameraMixin],
props: {
f7route: Object,
},
@@ -360,35 +179,13 @@
}
var loadServerSettings = localStorage.getItem('serverSettings')
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
var self = this
if (this.serverSettings && this.serverSettings.use) {
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
var xhr = new XMLHttpRequest()
xhr.open("GET", modelURL)
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.onload = function () {
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
var detectors = JSON.parse(xhr.response).detectors
var findLabel = detectors
.find( d => { return d.name == self.detectorName } )?.labels
.filter( l => { return l != "" } ).sort()
.map( l => { return {'name': l, 'detect': true} } )
self.detectorLabels = findLabel || []
}
xhr.send()
this.getRemoteLabels()
} else {
self.modelLoading = true
self.detectorLabels = self.classesList.map( l => { return {'name': l, 'detect': true} } )
self.loadModel(self.isCordova ? self.modelLocationCordova : self.modelLocation).then(() => {
self.modelLoading = false
this.modelLoading = true
this.detectorLabels = this.classesList.map( l => { return {'name': l, 'detect': true} } )
this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation).then(() => {
this.modelLoading = false
}).catch((e) => {
console.log(e.message)
f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
@@ -434,52 +231,21 @@
return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
},
setData () {
var self = this
if (this.serverSettings && this.serverSettings.use) {
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
var xhr = new XMLHttpRequest()
xhr.open("POST", modelURL)
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.onload = function () {
self.detecting = false
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
self.resultData = JSON.parse(xhr.response)
self.uploadDirty = true
}
var doodsData = {
"detector_name": this.detectorName,
"detect": {
"*": 1
},
"data": this.imageView.src.split(',')[1]
}
xhr.send(JSON.stringify(doodsData))
this.remoteDetect()
} else {
this.localDetect(this.imageView).then(dets => {
self.detecting = false
self.resultData = dets
self.uploadDirty = true
this.detecting = false
this.resultData = dets
this.uploadDirty = true
}).catch((e) => {
console.log(e.message)
self.detecting = false
self.resultData = {}
this.detecting = false
this.resultData = {}
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
})
}
},
remoteTimeout () {
this.detecting = false
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
},
selectAll (ev) {
if (ev.target.checked) {
this.detectorLabels.forEach( s => s.detect = true )
@@ -494,29 +260,10 @@
return
}
if (mode == "camera") {
const devicesList = await navigator.mediaDevices.enumerateDevices()
this.videoDeviceAvailable = devicesList.some( d => d.kind == "videoinput")
if (this.videoDeviceAvailable) {
navigator.mediaDevices.getUserMedia({video: true})
var vidConstraint = {
video: {
width: {
ideal: 1920
},
height: {
ideal: 1080
},
facingMode: 'environment'
},
audio: false
}
const stream = await navigator.mediaDevices.getUserMedia(vidConstraint);
this.videoAvailable = true
this.cameraStream = stream
return
}
this.videoAvailable = await this.openCamera()
if (this.videoAvailable) { return }
}
var loadResult = this.$refs.image_chooser.click()
this.$refs.image_chooser.click()
},
onFail (message) {
alert(`Camera fail: ${message}`)
@@ -565,8 +312,7 @@
getImage (searchImage) {
let loadImage = new Promise(resolve => {
if (this.videoAvailable) {
this.cameraStream.getTracks().forEach( t => t.stop())
this.videoAvailable = false
this.closeCamera()
this.detecting = true
resolve(searchImage)
} else if (this.isCordova && this.imageLoadMode == "camera") {
@@ -604,19 +350,6 @@
f7.dialog.alert(`Error loading image: ${e.message}`)
})
},
async videoStream() {
//TODO
},
captureVidFrame() {
const vidViewer = this.$refs.vid_viewer
vidViewer.pause()
let tempCVS = document.createElement('canvas')
tempCVS.height = vidViewer.videoHeight || parseInt(vidViewer.style.height)
tempCVS.width = vidViewer.videoWidth || parseInt(vidViewer.style.width)
const tempCtx = tempCVS.getContext('2d')
tempCtx.drawImage(vidViewer, 0, 0)
this.getImage(tempCVS.toDataURL())
},
async submitData () {
var uploadData = this.showResults
.filter( d => { return d.aboveThreshold && d.isSearched && !d.isDeleted })

View File

@@ -1,4 +1,5 @@
import * as tf from '@tensorflow/tfjs'
import { f7 } from 'framework7-vue'
var model = null
@@ -77,6 +78,70 @@ export default {
console.timeEnd('post-process')
return output
},
getRemoteLabels() {
var self = this
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
var xhr = new XMLHttpRequest()
xhr.open("GET", modelURL)
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.onload = function () {
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return
}
var detectors = JSON.parse(xhr.response).detectors
var findLabel = detectors
.find( d => { return d.name == self.detectorName } )?.labels
.filter( l => { return l != "" } ).sort()
.map( l => { return {'name': l, 'detect': true} } )
self.detectorLabels = findLabel || []
}
xhr.onerror = function (e) {
f7.dialog.alert('ALVINN has encountered an unknown server error')
return
}
xhr.send()
},
remoteDetect() {
var self = this
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
var xhr = new XMLHttpRequest()
xhr.open("POST", modelURL)
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.onload = function () {
self.detecting = false
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
self.resultData = JSON.parse(xhr.response)
self.uploadDirty = true
}
var doodsData = {
"detector_name": this.detectorName,
"detect": {
"*": 1
},
"data": this.imageView.src.split(',')[1]
}
xhr.send(JSON.stringify(doodsData))
},
remoteTimeout () {
this.detecting = false
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
}
}
}