Justin Georgi
619211e827
Closes: #48 This PR should allow the user to upload an image that has been taken with the camera and the associated detection information to a file server. These file can then be used to add data to ALVINN's models. Current upload destination is set to a file drop folder in Georgi Lab Nextcloud, but eventually (maybe) should be changed to some dedicated ALVINN backend. Also, prediction data is still not formatted in any specific way. It is just a JSON array of (top, left, bottom, right, label) objects. Reviewed-on: Georgi_Lab/ALVINN_f7#56
468 lines
16 KiB
Vue
468 lines
16 KiB
Vue
<template>
|
|
<f7-page name="detect">
|
|
<!-- Top Navbar -->
|
|
<f7-navbar :sliding="false" back-link="true" back-link-url="/" back-link-force>
|
|
<f7-nav-title sliding>{{ regions[activeRegion] }}</f7-nav-title>
|
|
<f7-nav-right>
|
|
<f7-link icon-ios="f7:menu" icon-md="material:settings" :panel-open="`#${detectorName}-settings`"></f7-link>
|
|
</f7-nav-right>
|
|
</f7-navbar>
|
|
<f7-block class="detect-grid">
|
|
<div class="image-container">
|
|
<img v-if="imageView" :src="imageView" id="im-display" ref="image_src" style="flex: 1 1 0%; object-fit: contain; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0;" />
|
|
<SvgIcon v-else icon="image" fill-color="var(--avn-theme-color)" @click="selectImage" />
|
|
<div ref="structure_box" style="border: solid 3px var(--avn-structure-box-color); position: absolute; display: none; box-sizing: border-box;" />
|
|
</div>
|
|
<div v-if="resultData && resultData.detections" class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
|
|
<f7-chip v-for="result in showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted })"
|
|
:class="(result.resultIndex == selectedChip) ? 'selected-chip' : ''"
|
|
:text="result.label"
|
|
media=" "
|
|
:tooltip="result.confidence.toFixed(1)"
|
|
deleteable
|
|
@click="selectChip(result.resultIndex)"
|
|
@delete="deleteChip(result.resultIndex)"
|
|
:style="chipGradient(result.confidence)"
|
|
/>
|
|
<span v-if="numResults == 0" style="height: var(--f7-chip-height); font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">No results.</span>
|
|
</div>
|
|
<f7-segmented class="image-menu" raised>
|
|
<f7-button popover-open="#region-popover">
|
|
<RegionIcon :region="activeRegion" />
|
|
</f7-button>
|
|
<f7-button popover-open="#capture-popover">
|
|
<SvgIcon icon="image"/>
|
|
</f7-button>
|
|
<f7-button @click="setData" :class="(imageLoaded) ? '' : 'disabled'">
|
|
<SvgIcon icon="visibility"/>
|
|
</f7-button>
|
|
<f7-button :class="(numResults && uploadDirty && viewedAll) ? '' : 'disabled'" @click="submitData">
|
|
<SvgIcon :icon="(uploadUid) ? 'cloud_done' : 'cloud_upload'"/>
|
|
</f7-button>
|
|
</f7-segmented>
|
|
<input type="file" ref="image_chooser" @change="getImage()" accept="image/*" capture="environment" style="display: none;"/>
|
|
</f7-block>
|
|
|
|
<f7-panel :id="detectorName + '-settings'" right cover>
|
|
<f7-page>
|
|
<f7-navbar title="Detection Settings"></f7-navbar>
|
|
<f7-list>
|
|
<f7-list-input v-model:value="detectorLevel" :label="`Confidence % threshold: ${detectorLevel}`" type="range" />
|
|
<f7-list-item accordion-item title="Structures">
|
|
<f7-accordion-content>
|
|
<f7-list>
|
|
<f7-list-item checkbox checked checkbox-icon="end" title="All/none" @change="selectAll"></f7-list-item>
|
|
<f7-list-item v-for="structure in detectorLabels" :key="structure.name" checkbox checkbox-icon="end" v-model:checked="structure.detect" :title="structure.name"></f7-list-item>
|
|
</f7-list>
|
|
</f7-accordion-content>
|
|
</f7-list-item>
|
|
<f7-list-item title="Turn on debugging">
|
|
<f7-toggle v-model:checked="debugOn" style="margin-right: 16px;" />
|
|
</f7-list-item>
|
|
</f7-list>
|
|
<f7-block v-if="debugOn" v-html="debugDisplay" />
|
|
</f7-page>
|
|
</f7-panel>
|
|
|
|
<f7-popover id="region-popover">
|
|
<f7-segmented raised style="flex-wrap: wrap; flex-direction: column;">
|
|
<f7-button style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
|
|
<RegionIcon :region="0" />
|
|
</f7-button>
|
|
<f7-button style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
|
|
<RegionIcon :region="1" />
|
|
</f7-button>
|
|
<f7-button style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
|
|
<RegionIcon :region="2" />
|
|
</f7-button>
|
|
<f7-button class="disabled" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
|
|
<RegionIcon :region="3" />
|
|
</f7-button>
|
|
</f7-segmented>
|
|
</f7-popover>
|
|
|
|
<f7-popover id="capture-popover">
|
|
<f7-segmented raised style="flex-wrap: wrap; flex-direction: column;">
|
|
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" class="disabled" @click="videoStream">
|
|
<SvgIcon icon="videocam"/>
|
|
</f7-button>
|
|
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('camera')">
|
|
<SvgIcon icon="photo_camera" />
|
|
</f7-button>
|
|
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('file')">
|
|
<SvgIcon icon="photo_library" />
|
|
</f7-button>
|
|
</f7-segmented>
|
|
</f7-popover>
|
|
|
|
</f7-page>
|
|
</template>
|
|
|
|
<style>
|
|
.detect-grid {
|
|
display: grid;
|
|
grid-template-columns: 1fr;
|
|
grid-template-rows: 1fr auto min-content;
|
|
grid-template-areas:
|
|
"image-view"
|
|
"result-view"
|
|
"menu-view";
|
|
justify-items: center;
|
|
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
|
|
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
|
|
}
|
|
|
|
.image-container {
|
|
grid-area: image-view;
|
|
max-width: 100%;
|
|
max-height: 100%;
|
|
min-width: 0;
|
|
min-height: 0;
|
|
position: relative;
|
|
display: flex;
|
|
align-self: stretch;
|
|
}
|
|
|
|
.chip-media {
|
|
background-color: var(--chip-media-background) !important;
|
|
}
|
|
|
|
.chip-results {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 5px;
|
|
padding: 10px;
|
|
--f7-chip-border-radius: 16px;
|
|
--f7-chip-media-size: 32px;
|
|
--f7-chip-font-weight: normal;
|
|
}
|
|
|
|
.chip-results .chip {
|
|
padding-left: 8px;
|
|
}
|
|
|
|
.selected-chip {
|
|
font-weight: 500;
|
|
box-shadow: 4px 4px 1px var(--avn-theme-color);
|
|
transform: translate(-2px, -2px);
|
|
}
|
|
|
|
.image-menu {
|
|
grid-area: menu-view;
|
|
margin: 5px;
|
|
/*width: 90%;*/
|
|
max-width: 400px;
|
|
min-width: 192px;
|
|
}
|
|
|
|
.image-menu .button {
|
|
aspect-ratio: 1;
|
|
height: auto;
|
|
padding: 5px;
|
|
flex: 1 1 0%;
|
|
}
|
|
|
|
.button > svg {
|
|
aspect-ratio: 1;
|
|
height: auto;
|
|
width: 100%;
|
|
}
|
|
|
|
@media (max-height: 450px) and (orientation: landscape) {
|
|
.detect-grid {
|
|
grid-template-columns: minmax(0,1fr) max-content auto;
|
|
grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
|
|
grid-template-areas:
|
|
"image-view result-view menu-view";
|
|
justify-items: stretch;
|
|
align-items: stretch;
|
|
height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
|
|
max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
|
|
position: relative;
|
|
}
|
|
|
|
.chip-results {
|
|
flex-direction: column;
|
|
max-height: 100%;
|
|
justify-self: start;
|
|
flex-wrap: nowrap;
|
|
overflow-y: scroll;
|
|
}
|
|
|
|
.image-container {
|
|
flex-direction: column;
|
|
}
|
|
|
|
.image-menu {
|
|
flex-direction: column;
|
|
aspect-ratio: .25;
|
|
width: auto;
|
|
min-width: 0;
|
|
height: 100%;
|
|
}
|
|
|
|
.image-menu .button {
|
|
aspect-ratio: 1;
|
|
width: auto;
|
|
height: 100%;
|
|
flex: 1 1 0%;
|
|
border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
|
|
border-bottom-left-radius: 0px !important;
|
|
}
|
|
|
|
.button > svg {
|
|
width:auto;
|
|
height: 100%;
|
|
}
|
|
}
|
|
</style>
|
|
|
|
<script>
|
|
import { f7 } from 'framework7-vue'
|
|
|
|
import RegionIcon from '../components/region-icon.vue'
|
|
import SvgIcon from '../components/svg-icon.vue'
|
|
|
|
import submitMixin from './submit-mixin'
|
|
|
|
export default {
|
|
mixins: [submitMixin],
|
|
props: {
|
|
f7route: Object,
|
|
},
|
|
components: {
|
|
RegionIcon,
|
|
SvgIcon
|
|
},
|
|
data () {
|
|
return {
|
|
regions: ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck'],
|
|
resultData: {},
|
|
selectedChip: -1,
|
|
activeRegion: 4,
|
|
imageRegion: '',
|
|
imageLoaded: false,
|
|
imageView: '',
|
|
imageLoadMode: "environment",
|
|
detectorName: '',
|
|
detectorLevel: 50,
|
|
detectorLabels: [],
|
|
serverSettings: {},
|
|
debugOn: false,
|
|
debugText: ['Variables loaded'],
|
|
isCordova: !!window.cordova,
|
|
uploadUid: null,
|
|
uploadDirty: false
|
|
}
|
|
},
|
|
created () {
|
|
switch (this.f7route.params.region) {
|
|
case 'thorax':
|
|
this.activeRegion = 0
|
|
this.detectorName = 'thorax'
|
|
break;
|
|
case 'abdomen':
|
|
this.activeRegion = 1
|
|
this.detectorName = 'combined'
|
|
break;
|
|
case 'limbs':
|
|
this.activeRegion = 2
|
|
this.detectorName = 'defaultNew'
|
|
break;
|
|
case 'head':
|
|
this.activeRegion = 3
|
|
break;
|
|
}
|
|
var loadServerSettings = localStorage.getItem('serverSettings')
|
|
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
|
|
var self = this
|
|
if (this.serverSettings && this.serverSettings.use) {
|
|
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
|
|
var xhr = new XMLHttpRequest()
|
|
xhr.open("GET", modelURL)
|
|
xhr.setRequestHeader('Content-Type', 'application/json')
|
|
xhr.onload = function () {
|
|
if (self.debugOn) self.debugText.push(xhr.response) //DEBUG ANDROID BUILD
|
|
if (this.status !== 200) {
|
|
//this.response.text().then(function(message){alert(message)})
|
|
self.debugText.push(xhr.response)
|
|
console.log(xhr.response)
|
|
return;
|
|
}
|
|
var detectors = JSON.parse(xhr.response).detectors
|
|
var findLabel = detectors
|
|
.find( d => { return d.name == self.detectorName } )?.labels
|
|
.filter( l => { return l != "" } ).sort()
|
|
.map( l => { return {'name': l, 'detect': true} } )
|
|
self.detectorLabels = findLabel || []
|
|
}
|
|
xhr.send()
|
|
}
|
|
},
|
|
deactivated () {
|
|
console.log('destroy the panel!')
|
|
},
|
|
computed: {
|
|
debugDisplay () {
|
|
return this.debugText.join('<br/>')
|
|
},
|
|
showResults () {
|
|
var filteredResults = this.resultData.detections
|
|
if (!filteredResults) return []
|
|
var allSelect = this.detectorLabels.every( s => { return s.detect } )
|
|
var selectedLabels = this.detectorLabels
|
|
.filter( l => { return l.detect })
|
|
.map( l => { return l.name })
|
|
filteredResults.forEach( (d, i) => {
|
|
filteredResults[i].resultIndex = i
|
|
filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel
|
|
filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label)
|
|
})
|
|
return filteredResults
|
|
},
|
|
numResults () {
|
|
return this.showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted }).length
|
|
},
|
|
viewedAll () {
|
|
return this.resultData.detections
|
|
.filter( s => { return s.confidence >= this.detectorLevel})
|
|
.every( s => { return s.beenViewed })
|
|
}
|
|
},
|
|
methods: {
|
|
chipGradient (confVal) {
|
|
return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
|
|
},
|
|
setData () {
|
|
var self = this
|
|
this.resetView()
|
|
if (this.serverSettings && this.serverSettings.use) {
|
|
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
|
|
var xhr = new XMLHttpRequest()
|
|
xhr.open("POST", modelURL)
|
|
xhr.setRequestHeader('Content-Type', 'application/json')
|
|
xhr.onload = function () {
|
|
if (self.debugOn) self.debugText.push(xhr.response)
|
|
if (this.status !== 200) {
|
|
//this.response.text().then(function(message){alert(message)})
|
|
self.debugText.push(xhr.response)
|
|
console.log(xhr.response)
|
|
return;
|
|
}
|
|
self.resultData = JSON.parse(xhr.response)
|
|
self.uploadDirty = true
|
|
}
|
|
|
|
var doodsData = {
|
|
"detector_name": this.detectorName,
|
|
"detect": {
|
|
"*": 1
|
|
},
|
|
"data": this.imageView.split(',')[1]
|
|
}
|
|
|
|
xhr.send(JSON.stringify(doodsData))
|
|
} else {
|
|
//TODO
|
|
f7.dialog.alert('Using built-in model')
|
|
}
|
|
},
|
|
selectAll (ev) {
|
|
if (ev.target.checked) {
|
|
this.detectorLabels.forEach( s => s.detect = true )
|
|
} else {
|
|
this.detectorLabels.forEach( s => s.detect = false )
|
|
}
|
|
},
|
|
selectImage (mode) {
|
|
this.imageLoadMode = mode
|
|
if (mode == "camera") {
|
|
this.$refs.image_chooser.setAttribute("capture","environment")
|
|
} else {
|
|
this.$refs.image_chooser.removeAttribute("capture")
|
|
}
|
|
if (this.isCordova && mode == "camera") {
|
|
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
|
|
} else {
|
|
var loadResult = this.$refs.image_chooser.click()
|
|
}
|
|
},
|
|
onFail (message) {
|
|
alert(`Camera fail: ${message}`)
|
|
},
|
|
selectChip ( iChip ) {
|
|
if (this.selectedChip == iChip) {
|
|
this.resetView()
|
|
return
|
|
}
|
|
|
|
var imgWidth
|
|
var imgHeight
|
|
|
|
this.selectedChip = iChip
|
|
const box = this.$refs.structure_box
|
|
const img = this.$refs.image_src
|
|
var imgAspect = img.naturalWidth / img.naturalHeight
|
|
var rendAspect = img.offsetWidth / img.offsetHeight
|
|
if (imgAspect >= rendAspect) {
|
|
imgWidth = img.offsetWidth
|
|
imgHeight = img.offsetWidth / imgAspect
|
|
} else {
|
|
imgWidth = img.offsetHeight * imgAspect
|
|
imgHeight = img.offsetHeight
|
|
}
|
|
box.style.display = 'block'
|
|
box.style.left = `${(img.offsetWidth - imgWidth) / 2 + this.resultData.detections[iChip].left * imgWidth}px`
|
|
box.style.top = `${(img.offsetHeight - imgHeight) / 2 + this.resultData.detections[iChip].top * imgHeight}px`
|
|
box.style.width = `${(Math.min(this.resultData.detections[iChip].right, 1) - Math.max(this.resultData.detections[iChip].left, 0)) * imgWidth}px`
|
|
box.style.height = `${(Math.min(this.resultData.detections[iChip].bottom, 1) - Math.max(this.resultData.detections[iChip].top, 0)) * imgHeight}px`
|
|
this.resultData.detections[iChip].beenViewed = true
|
|
},
|
|
deleteChip ( iChip ) {
|
|
f7.dialog.confirm(`${this.resultData.detections[iChip].label} is identified with ${this.resultData.detections[iChip].confidence.toFixed(1)}% confidence. Are you sure you want to delete it?`, () => {
|
|
this.resultData.detections.splice(iChip, 1)
|
|
this.resetView()
|
|
this.uploadDirty = true
|
|
});
|
|
},
|
|
getImage (searchImage) {
|
|
var self = this
|
|
let loadImage =new Promise(resolve => {
|
|
if (this.isCordova && this.imageLoadMode == "camera") {
|
|
this.imageView = 'data:image/jpg;base64,' + searchImage
|
|
} else {
|
|
const searchImage = this.$refs.image_chooser.files[0]
|
|
var reader = new FileReader()
|
|
reader.addEventListener("loadend", () => {this.imageView = reader.result})
|
|
reader.readAsDataURL(searchImage)
|
|
}
|
|
resolve()
|
|
})
|
|
loadImage.then(() => {
|
|
this.imageLoaded = true
|
|
this.resultData = {}
|
|
this.resetView()
|
|
}).catch((e) => {
|
|
console.log(e.message)
|
|
f7.dialog.alert(`Error loading image: ${e.message}`)
|
|
})
|
|
},
|
|
resetView() {
|
|
this.selectedChip = -1
|
|
const box = this.$refs.structure_box
|
|
box.style.display = 'none'
|
|
},
|
|
videoStream() {
|
|
//TODO
|
|
return null
|
|
},
|
|
async submitData () {
|
|
var uploadData = this.showResults
|
|
.filter( d => { return d.aboveThreshold && d.isSearched && !d.isDeleted })
|
|
.map( r => { return {"top": r.top, "left": r.left, "bottom": r.bottom, "right": r.right, "label": r.label}})
|
|
this.uploadUid = await this.uploadData(this.imageView.split(',')[1],uploadData,this.uploadUid)
|
|
if (this.uploadUid) { this.uploadDirty = false }
|
|
}
|
|
}
|
|
}
|
|
</script> |