Files
ALVINN_f7/src/pages/detect.vue
2024-02-20 20:08:16 -07:00

625 lines
22 KiB
Vue

<template>
<f7-page name="detect" :id="detectorName + '-detect-page'">
<!-- Top Navbar -->
<f7-navbar :sliding="false" :back-link="true" back-link-url="/" back-link-force>
<f7-nav-title sliding>{{ regions[activeRegion] }}</f7-nav-title>
</f7-navbar>
<f7-block class="detect-grid">
<div class="image-container">
<canvas id="im-draw" ref="image_cvs" @click="structureClick" :style="`display: ${imageLoaded ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat`" />
<SvgIcon v-if="!imageView" :icon="f7route.params.region" fill-color="var(--avn-theme-color)" @click="selectImage" />
</div>
<div v-if="(resultData && resultData.detections) || detecting" class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
<f7-chip v-for="result in showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted })"
:class="(result.resultIndex == selectedChip) ? 'selected-chip' : ''"
:text="result.label"
media=" "
:tooltip="result.confidence.toFixed(1)"
deleteable
@click="selectChip(result.resultIndex)"
@delete="deleteChip(result.resultIndex)"
:style="chipGradient(result.confidence)"
/>
<span v-if="numResults == 0 && !detecting" style="height: var(--f7-chip-height); font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">No results.</span>
<f7-preloader v-if="detecting || modelLoading" size="32" style="color: var(--avn-theme-color);" />
</div>
<div v-if="showDetectSettings" class="detect-inputs" style="grid-area: detect-settings;">
<f7-range class="level-slide-horz" :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
<f7-range class="level-slide-vert" vertical :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
<f7-button @click="() => detectPanel = !detectPanel" :panel-open="!detectPanel && `#${detectorName}-settings`" :panel-close="detectPanel && `#${detectorName}-settings`" style="flex: 0 1 20%">
<SvgIcon icon="check_list"/>
</f7-button>
<f7-button @click="setData" style="flex: 0 1 20%">
<SvgIcon icon="refresh_search"/>
</f7-button>
</div>
<f7-segmented class="image-menu" raised>
<f7-button popover-open="#region-popover">
<RegionIcon :region="activeRegion" />
</f7-button>
<f7-button popover-open="#capture-popover">
<SvgIcon icon="camera_add"/>
</f7-button>
<f7-button @click="() => showDetectSettings = !showDetectSettings" :class="(imageLoaded) ? '' : 'disabled'">
<SvgIcon icon="visibility"/>
</f7-button>
<f7-button :class="(numResults && uploadDirty && viewedAll) ? '' : 'disabled'" @click="submitData">
<SvgIcon :icon="(uploadUid) ? 'cloud_done' : 'cloud_upload'"/>
</f7-button>
</f7-segmented>
<input type="file" ref="image_chooser" @change="getImage()" accept="image/*" capture="environment" style="display: none;"/>
</f7-block>
<f7-panel :id="detectorName + '-settings'" right cover :backdrop="false" :container-el="`#${detectorName}-detect-page`">
<f7-page>
<f7-navbar title="Detection Settings"></f7-navbar>
<f7-list>
<f7-list-button title="Close List" @click="() => detectPanel = false" :panel-close="`#${detectorName}-settings`"></f7-list-button>
<f7-list-item checkbox checked checkbox-icon="end" title="All/none" @change="selectAll"></f7-list-item>
<f7-list-item v-for="structure in detectorLabels" :key="structure.name" checkbox checkbox-icon="end" v-model:checked="structure.detect" :title="structure.name"></f7-list-item>
</f7-list>
</f7-page>
</f7-panel>
<f7-popover id="region-popover" class="popover-button-menu">
<f7-segmented raised class="segment-button-menu">
<f7-button :class="(getRegions.includes('thorax')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
<RegionIcon :region="0" />
</f7-button>
<f7-button :class="(getRegions.includes('abdomen')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
<RegionIcon :region="1" />
</f7-button>
<f7-button :class="(getRegions.includes('limbs')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
<RegionIcon :region="2" />
</f7-button>
<f7-button :class="(getRegions.includes('head')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
<RegionIcon :region="3" />
</f7-button>
</f7-segmented>
</f7-popover>
<f7-popover id="capture-popover" class="popover-button-menu">
<f7-segmented raised class="segment-button-menu">
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('camera')">
<SvgIcon icon="photo_camera" />
</f7-button>
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('file')">
<SvgIcon icon="photo_library" />
</f7-button>
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" class="disabled" @click="videoStream">
<SvgIcon icon="videocam"/>
</f7-button>
</f7-segmented>
</f7-popover>
</f7-page>
</template>
<style>
.detect-grid {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr 56px auto min-content;
grid-template-areas:
"image-view"
"result-view"
"detect-settings"
"menu-view";
justify-items: center;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
}
.image-container {
grid-area: image-view;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
position: relative;
display: flex;
align-self: stretch;
}
.popover-button-menu {
max-width: 90vw;
max-height: 90vh;
width: auto;
}
.segment-button-menu {
flex-wrap: nowrap;
flex-direction: column;
max-height: 100%;
min-height: 0px;
}
.chip-media {
background-color: var(--chip-media-background) !important;
}
.chip-results {
display: flex;
flex-wrap: wrap;
gap: 5px;
padding: 10px;
--f7-chip-border-radius: 16px;
--f7-chip-media-size: 32px;
--f7-chip-font-weight: normal;
}
.chip-results .chip {
padding-left: 8px;
}
.selected-chip {
font-weight: 500;
box-shadow: 4px 4px 1px var(--avn-theme-color);
transform: translate(-2px, -2px);
}
.detect-inputs {
display: flex;
align-items: center;
margin: 5px;
width: 100%;
max-width: 400px;
min-width: 192px;
}
.level-slide-vert {
display: none;
}
.image-menu {
grid-area: menu-view;
margin: 5px;
/*width: 90%;*/
max-width: 400px;
min-width: 192px;
}
.image-menu .button {
aspect-ratio: 1;
height: auto;
padding: 5px;
flex: 1 1 0%;
}
.image-menu > .button > svg {
aspect-ratio: 1;
height: auto;
width: 100%;
}
.segment-button-menu .button {
padding: 8px;
aspect-ratio: 1;
width: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
@media (max-height: 450px) and (orientation: landscape) {
.detect-grid {
grid-template-columns: minmax(0,1fr) minmax(56px,max-content) auto auto;
grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
grid-template-areas:
"image-view result-view detect-settings menu-view";
justify-items: stretch;
align-items: stretch;
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
position: relative;
}
.chip-results {
flex-direction: column;
max-height: 100%;
justify-self: start;
flex-wrap: nowrap;
overflow-y: scroll;
}
.detect-inputs {
flex-direction: column;
min-width: 0;
max-width: 72px;
}
.level-slide-horz {
display: none;
}
.level-slide-vert {
display: block;
}
.image-container {
flex-direction: column;
}
.image-menu {
flex-direction: column;
aspect-ratio: .25;
width: auto;
min-width: 0;
height: 100%;
}
.image-menu .button {
aspect-ratio: 1;
width: auto;
height: 100%;
flex: 1 1 0%;
border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
border-bottom-left-radius: 0px !important;
}
.segment-button-menu {
flex-direction: row;
max-height: 100%;
min-height: 0px;
}
.segment-button-menu .button {
height: auto;
flex: 1 1 0%;
max-height: 100px;
max-width: 100px;
}
.button > svg {
width: 100%;
height: auto;
}
}
</style>
<script>
import { f7 } from 'framework7-vue'
import store from '../js/store'
import RegionIcon from '../components/region-icon.vue'
import SvgIcon from '../components/svg-icon.vue'
import submitMixin from './submit-mixin'
import detectMixin from './local-detect'
import thoraxClasses from '../models/thorax_tfwm/classes.json'
export default {
mixins: [submitMixin, detectMixin],
props: {
f7route: Object,
},
components: {
RegionIcon,
SvgIcon
},
data () {
return {
regions: ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck'],
resultData: {},
selectedChip: -1,
activeRegion: 4,
classesList: [],
imageLoaded: false,
imageView: null,
imageLoadMode: "environment",
detecting: false,
detectPanel: false,
showDetectSettings: false,
detectorName: '',
detectorLevel: 50,
detectorLabels: [],
serverSettings: {},
isCordova: !!window.cordova,
uploadUid: null,
uploadDirty: false,
modelLocation: '',
modelLoading: false
}
},
setup() {
return store()
},
created () {
switch (this.f7route.params.region) {
case 'thorax':
this.activeRegion = 0
this.detectorName = 'thorax'
this.classesList = thoraxClasses
this.modelLocation = '../models/thorax_tfwm/model.json'
this.modelLocationCordova = 'https://localhost/models/thorax_tfwm/model.json'
break;
case 'abdomen':
this.activeRegion = 1
this.detectorName = 'combined'
break;
case 'limbs':
this.activeRegion = 2
this.detectorName = 'defaultNew'
break;
case 'head':
this.activeRegion = 3
break;
}
var loadServerSettings = localStorage.getItem('serverSettings')
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
var self = this
if (this.serverSettings && this.serverSettings.use) {
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
var xhr = new XMLHttpRequest()
xhr.open("GET", modelURL)
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.onload = function () {
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
var detectors = JSON.parse(xhr.response).detectors
var findLabel = detectors
.find( d => { return d.name == self.detectorName } )?.labels
.filter( l => { return l != "" } ).sort()
.map( l => { return {'name': l, 'detect': true} } )
self.detectorLabels = findLabel || []
}
xhr.send()
} else {
self.modelLoading = true
self.detectorLabels = self.classesList.map( l => { return {'name': l, 'detect': true} } )
self.loadModel(self.isCordova ? self.modelLocationCordova : self.modelLocation).then(() => {
self.modelLoading = false
}).catch((e) => {
console.log(e.message)
f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
})
}
window.onresize = (e) => { this.selectChip('redraw') }
},
computed: {
showResults () {
var filteredResults = this.resultData.detections
if (!filteredResults) return []
var allSelect = this.detectorLabels.every( s => { return s.detect } )
var selectedLabels = this.detectorLabels
.filter( l => { return l.detect })
.map( l => { return l.name })
filteredResults.forEach( (d, i) => {
filteredResults[i].resultIndex = i
filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel
filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label)
})
if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) {
this.selectChip(this.selectedChip)
}
return filteredResults
},
numResults () {
return this.showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted }).length
},
viewedAll () {
if (this.resultData && this.resultData.detections) {
return this.resultData.detections
.filter( s => { return s.confidence >= this.detectorLevel})
.every( s => { return s.beenViewed })
} else {
return false
}
}
},
methods: {
chipGradient (confVal) {
return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
},
setData () {
var self = this
if (this.serverSettings && this.serverSettings.use) {
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
var xhr = new XMLHttpRequest()
xhr.open("POST", modelURL)
xhr.timeout = 10000
xhr.ontimeout = this.remoteTimeout
xhr.setRequestHeader('Content-Type', 'application/json')
xhr.onload = function () {
self.detecting = false
if (this.status !== 200) {
console.log(xhr.response)
const errorResponse = JSON.parse(xhr.response)
f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
return;
}
self.resultData = JSON.parse(xhr.response)
self.uploadDirty = true
}
var doodsData = {
"detector_name": this.detectorName,
"detect": {
"*": 1
},
"data": this.imageView.src.split(',')[1]
}
xhr.send(JSON.stringify(doodsData))
} else {
this.localDetect(this.imageView).then(dets => {
self.detecting = false
self.resultData = dets
self.uploadDirty = true
}).catch((e) => {
console.log(e.message)
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
})
}
},
remoteTimeout () {
this.detecting = false
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
},
selectAll (ev) {
if (ev.target.checked) {
this.detectorLabels.forEach( s => s.detect = true )
} else {
this.detectorLabels.forEach( s => s.detect = false )
}
},
selectImage (mode) {
this.imageLoadMode = mode
if (mode == "camera") {
this.$refs.image_chooser.setAttribute("capture","environment")
} else {
this.$refs.image_chooser.removeAttribute("capture")
}
if (this.isCordova && mode == "camera") {
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
} else {
var loadResult = this.$refs.image_chooser.click()
}
},
onFail (message) {
alert(`Camera fail: ${message}`)
},
selectChip ( iChip ) {
const [imCanvas, imageCtx] = this.resetView()
if (this.selectedChip == iChip) {
this.selectedChip = -1
return
}
if (iChip == 'redraw') {
if (this.selectedChip == -1) return
iChip = this.selectedChip
}
const boxCoords = this.box2cvs(this.resultData.detections[iChip])[0]
var boxLeft = boxCoords.cvsLeft
var boxTop = boxCoords.cvsTop
var boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
var boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
imageCtx.strokeRect(boxLeft,boxTop,boxWidth,boxHeight)
this.selectedChip = iChip
this.resultData.detections[iChip].beenViewed = true
},
deleteChip ( iChip ) {
f7.dialog.confirm(`${this.resultData.detections[iChip].label} is identified with ${this.resultData.detections[iChip].confidence.toFixed(1)}% confidence. Are you sure you want to delete it?`, () => {
this.resetView()
this.resultData.detections.splice(iChip, 1)
this.selectedChip = -1
this.uploadDirty = true
});
},
resetView () {
const imCanvas = this.$refs.image_cvs
const imageCtx = imCanvas.getContext("2d")
imCanvas.width = imCanvas.clientWidth
imCanvas.height = imCanvas.clientHeight
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
imageCtx.strokeStyle = 'yellow'
imageCtx.lineWidth = 3
return [imCanvas, imageCtx]
},
getImage (searchImage) {
let loadImage =new Promise(resolve => {
if (this.isCordova && this.imageLoadMode == "camera") {
resolve('data:image/jpg;base64,' + searchImage)
} else {
const searchImage = this.$refs.image_chooser.files[0]
var reader = new FileReader()
reader.addEventListener("loadend", () => {
this.detecting = true
resolve(reader.result)
})
reader.readAsDataURL(searchImage)
}
})
loadImage.then((imgData) => {
this.imageLoaded = true
this.resultData = {}
this.imageView = new Image()
this.imageView.src = imgData
return(this.imageView.decode())
}).then( () => {
const [imCanvas, _] = this.resetView()
imCanvas.style['background-image'] = `url(${this.imageView.src})`
/******
* setTimeout is not a good solution,
* but it's the only way I can find to
* not cut off drawing of of the progress
* spinner
******/
setTimeout(() => {
this.setData()
}, 250)
}).catch((e) => {
console.log(e.message)
f7.dialog.alert(`Error loading image: ${e.message}`)
})
},
videoStream() {
//TODO
return null
},
async submitData () {
var uploadData = this.showResults
.filter( d => { return d.aboveThreshold && d.isSearched && !d.isDeleted })
.map( r => { return {"top": r.top, "left": r.left, "bottom": r.bottom, "right": r.right, "label": r.label}})
this.uploadUid = await this.uploadData(this.imageView.src.split(',')[1],uploadData,this.uploadUid)
if (this.uploadUid) { this.uploadDirty = false }
},
onLevelChange(value) {
this.detectorLevel = value
},
structureClick(e) {
const boxCoords = this.box2cvs(this.showResults)
var findBox = boxCoords.findIndex( (r, i) => { return r.cvsLeft <= e.offsetX &&
r.cvsRight >= e.offsetX &&
r.cvsTop <= e.offsetY &&
r.cvsBottom >= e.offsetY &&
this.resultData.detections[i].resultIndex > this.selectedChip &&
this.resultData.detections[i].aboveThreshold &&
this.resultData.detections[i].isSearched &&
!this.resultData.detections[i].isDeleted
})
this.selectChip(findBox >= 0 ? this.resultData.detections[findBox].resultIndex : this.selectedChip)
},
box2cvs(boxInput) {
if (!boxInput) return []
const boxList = boxInput.length ? boxInput : [boxInput]
const [imCanvas, imageCtx] = this.resetView()
var imgWidth
var imgHeight
const imgAspect = this.imageView.width / this.imageView.height
const rendAspect = imCanvas.width / imCanvas.height
if (imgAspect >= rendAspect) {
imgWidth = imCanvas.width
imgHeight = imCanvas.width / imgAspect
} else {
imgWidth = imCanvas.height * imgAspect
imgHeight = imCanvas.height
}
const cvsCoords = boxList.map( (d, i) => {
return {
"cvsLeft": (imCanvas.width - imgWidth) / 2 + d.left * imgWidth,
"cvsRight": (imCanvas.width - imgWidth) / 2 + d.right * imgWidth,
"cvsTop": (imCanvas.height - imgHeight) / 2 + d.top * imgHeight,
"cvsBottom": (imCanvas.height - imgHeight) / 2 + d.bottom * imgHeight
}
})
return cvsCoords
}
}
}
</script>