489 lines
21 KiB
Vue
489 lines
21 KiB
Vue
<template>
|
|
<f7-page name="detect" :id="detectorName + '-detect-page'">
|
|
<!-- Top Navbar -->
|
|
<f7-navbar :sliding="false" :back-link="true" back-link-url="/" back-link-force>
|
|
<f7-nav-title sliding>{{ regions[activeRegion] }}</f7-nav-title>
|
|
<f7-nav-right>
|
|
<f7-link v-if="!isCordova" :icon-only="true" tooltip="Fullscreen" :icon-f7="isFullscreen ? 'viewfinder_circle_fill' : 'viewfinder'" @click="toggleFullscreen"></f7-link>
|
|
<f7-link :icon-only="true" tooltip="ALVINN help" icon-f7="question_circle_fill" href="/help/"></f7-link>
|
|
</f7-nav-right>
|
|
</f7-navbar>
|
|
<f7-block class="detect-grid">
|
|
<div class="image-container" ref="image_container">
|
|
<SvgIcon v-if="!imageView && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)" @click="selectImage" />
|
|
<div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
|
|
<video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
|
|
<f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
|
|
</div>
|
|
<canvas id="im-draw" ref="image_cvs" @click="structureClick" :style="`display: ${(imageLoaded || videoAvailable) ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat; z-index: 2;`" />
|
|
</div>
|
|
<div class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
|
|
<f7-chip v-for="result in showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted })"
|
|
:class="(result.resultIndex == selectedChip) ? 'selected-chip' : ''"
|
|
:text="result.label"
|
|
media=" "
|
|
:tooltip="result.confidence.toFixed(1)"
|
|
deleteable
|
|
@click="selectChip(result.resultIndex)"
|
|
@delete="deleteChip(result.resultIndex)"
|
|
:style="chipGradient(result.confidence)"
|
|
/>
|
|
<div v-if="!numResults" style="height: var(--f7-chip-height); width: 100%; text-align: center; font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">{{ message }}</div>
|
|
<f7-progressbar v-if="(detecting || modelLoading)" style="width: 100%;" :infinite="true" />
|
|
</div>
|
|
<div v-if="showDetectSettings" class="detect-inputs" style="grid-area: detect-settings;">
|
|
<f7-range class="level-slide-horz" :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
|
|
<f7-range class="level-slide-vert" vertical :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
|
|
<f7-button @click="() => detectPanel = !detectPanel" :panel-open="!detectPanel && `#${detectorName}-settings`" :panel-close="detectPanel && `#${detectorName}-settings`" style="flex: 0 1 20%">
|
|
<SvgIcon icon="check_list"/>
|
|
</f7-button>
|
|
<f7-button @click="setData" style="flex: 0 1 20%">
|
|
<SvgIcon icon="refresh_search"/>
|
|
</f7-button>
|
|
</div>
|
|
<f7-segmented class="image-menu" raised>
|
|
<f7-button popover-open="#region-popover">
|
|
<RegionIcon :region="activeRegion" />
|
|
</f7-button>
|
|
<f7-button v-if="!videoAvailable" :class="(!modelLoading) ? '' : 'disabled'" popover-open="#capture-popover">
|
|
<SvgIcon icon="camera_add"/>
|
|
</f7-button>
|
|
<f7-button v-if="videoAvailable" @click="closeCamera()">
|
|
<SvgIcon icon="no_photography"/>
|
|
</f7-button>
|
|
<f7-button @click="() => showDetectSettings = !showDetectSettings" :class="(imageLoaded) ? '' : 'disabled'">
|
|
<SvgIcon icon="visibility"/>
|
|
<f7-badge v-if="numResults && (showResults.length != numResults)" color="red" style="position: absolute; right: 15%; top: 15%;">{{ showResults.length - numResults }}</f7-badge>
|
|
</f7-button>
|
|
<f7-button :class="(numResults && uploadDirty && viewedAll) ? '' : 'disabled'" @click="submitData">
|
|
<SvgIcon :icon="(uploadUid) ? 'cloud_done' : 'cloud_upload'"/>
|
|
</f7-button>
|
|
</f7-segmented>
|
|
<input type="file" ref="image_chooser" @change="getImage()" accept="image/*" style="display: none;"/>
|
|
<img src="../assets/target.svg" ref="target_image" style="display: none;" />
|
|
</f7-block>
|
|
|
|
<f7-panel :id="detectorName + '-settings'" right cover :backdrop="false" :container-el="`#${detectorName}-detect-page`">
|
|
<f7-page>
|
|
<f7-navbar title="Detection Settings"></f7-navbar>
|
|
<f7-list>
|
|
<f7-list-button title="Close List" @click="() => detectPanel = false" :panel-close="`#${detectorName}-settings`"></f7-list-button>
|
|
<f7-list-item checkbox checked checkbox-icon="end" title="All/none" @change="selectAll"></f7-list-item>
|
|
<f7-list-item v-for="structure in detectorLabels" :key="structure.name" checkbox checkbox-icon="end" v-model:checked="structure.detect" :title="structure.name"></f7-list-item>
|
|
</f7-list>
|
|
</f7-page>
|
|
</f7-panel>
|
|
|
|
<f7-popover id="region-popover" class="popover-button-menu">
|
|
<f7-segmented raised class="segment-button-menu">
|
|
<f7-button :class="(getRegions.includes('thorax')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
|
|
<RegionIcon :region="0" />
|
|
</f7-button>
|
|
<f7-button :class="(getRegions.includes('abdomen')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
|
|
<RegionIcon :region="1" />
|
|
</f7-button>
|
|
<f7-button :class="(getRegions.includes('limbs')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
|
|
<RegionIcon :region="2" />
|
|
</f7-button>
|
|
<f7-button :class="(getRegions.includes('head')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
|
|
<RegionIcon :region="3" />
|
|
</f7-button>
|
|
</f7-segmented>
|
|
</f7-popover>
|
|
|
|
<f7-popover id="capture-popover" class="popover-button-menu">
|
|
<f7-segmented raised class="segment-button-menu">
|
|
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('camera')">
|
|
<SvgIcon icon="photo_camera" />
|
|
</f7-button>
|
|
<f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('file')">
|
|
<SvgIcon icon="photo_library" />
|
|
</f7-button>
|
|
<f7-button v-if="demoEnabled" style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('sample')">
|
|
<SvgIcon icon="photo_sample"/>
|
|
</f7-button>
|
|
</f7-segmented>
|
|
</f7-popover>
|
|
|
|
</f7-page>
|
|
</template>
|
|
|
|
<style src="../css/detect.css" />
|
|
|
|
<script>
|
|
import { f7 } from 'framework7-vue'
|
|
|
|
import store from '../js/store'
|
|
import RegionIcon from '../components/region-icon.vue'
|
|
import SvgIcon from '../components/svg-icon.vue'
|
|
|
|
import submitMixin from './submit-mixin'
|
|
import detectionMixin from './detection-mixin'
|
|
import cameraMixin from './camera-mixin'
|
|
import { Conv2DBackpropFilter } from '@tensorflow/tfjs'
|
|
|
|
export default {
|
|
mixins: [submitMixin, detectionMixin, cameraMixin],
|
|
props: {
|
|
f7route: Object,
|
|
},
|
|
components: {
|
|
RegionIcon,
|
|
SvgIcon
|
|
},
|
|
data () {
|
|
return {
|
|
regions: ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck'],
|
|
resultData: {},
|
|
selectedChip: -1,
|
|
activeRegion: 4,
|
|
classesList: [],
|
|
imageLoaded: false,
|
|
imageView: null,
|
|
imageLoadMode: "environment",
|
|
detecting: false,
|
|
detectPanel: false,
|
|
showDetectSettings: false,
|
|
detectorName: '',
|
|
detectorLevel: 50,
|
|
detectorLabels: [],
|
|
serverSettings: {},
|
|
otherSettings: {},
|
|
isCordova: !!window.cordova,
|
|
isFullscreen: false,
|
|
uploadUid: null,
|
|
uploadDirty: false,
|
|
modelLocation: '',
|
|
miniLocation: '',
|
|
modelLoading: true,
|
|
reloadModel: false,
|
|
videoDeviceAvailable: false,
|
|
videoAvailable: false,
|
|
cameraStream: null
|
|
}
|
|
},
|
|
setup() {
|
|
return store()
|
|
},
|
|
created () {
|
|
let loadOtherSettings = localStorage.getItem('otherSettings')
|
|
if (loadOtherSettings) this.otherSettings = JSON.parse(loadOtherSettings)
|
|
let modelRoot = this.isCordova ? 'https://localhost' : '.'
|
|
this.detectorName = this.f7route.params.region
|
|
switch (this.detectorName) {
|
|
case 'thorax':
|
|
this.activeRegion = 0
|
|
break;
|
|
case 'abdomen':
|
|
this.activeRegion = 1
|
|
break;
|
|
case 'limbs':
|
|
this.activeRegion = 2
|
|
break;
|
|
case 'head':
|
|
this.activeRegion = 3
|
|
break;
|
|
}
|
|
this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
|
|
this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
|
|
fetch(`${this.isCordova ? 'https://localhost' : '.'}/models/${this.detectorName}/classes.json`)
|
|
.then((mod) => { return mod.json() })
|
|
.then((classes) => {
|
|
this.classesList = classes
|
|
this.detectorLabels = this.classesList.map( l => { return {'name': l, 'detect': true} } )
|
|
})
|
|
var loadServerSettings = localStorage.getItem('serverSettings')
|
|
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
|
|
},
|
|
mounted () {
|
|
if (this.serverSettings && this.serverSettings.use) {
|
|
this.getRemoteLabels()
|
|
this.modelLoading = false
|
|
} else {
|
|
this.modelLoading = true
|
|
this.loadModel(this.modelLocation, true).then(() => {
|
|
this.modelLoading = false
|
|
}).catch((e) => {
|
|
console.log(e.message)
|
|
f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
|
|
this.modelLoading = false
|
|
})
|
|
}
|
|
window.onresize = (e) => { this.selectChip('redraw') }
|
|
},
|
|
computed: {
|
|
message () {
|
|
if (this.modelLoading) {
|
|
return "Preparing ALVINN..."
|
|
} else if (this.detecting) {
|
|
return "Finding structures..."
|
|
} else if (this.numResults == 0 && this.imageLoaded) {
|
|
return "No results."
|
|
} else {
|
|
return "ALVINN is ready."
|
|
}
|
|
},
|
|
showResults () {
|
|
var filteredResults = this.resultData.detections
|
|
if (!filteredResults) return []
|
|
|
|
var allSelect = this.detectorLabels.every( s => { return s.detect } )
|
|
var selectedLabels = this.detectorLabels
|
|
.filter( l => { return l.detect })
|
|
.map( l => { return l.name })
|
|
filteredResults.forEach( (d, i) => {
|
|
filteredResults[i].resultIndex = i
|
|
filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel
|
|
filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label)
|
|
})
|
|
|
|
if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) {
|
|
this.selectChip(this.selectedChip)
|
|
}
|
|
return filteredResults
|
|
},
|
|
numResults () {
|
|
return this.showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted }).length
|
|
},
|
|
viewedAll () {
|
|
if (this.resultData && this.resultData.detections) {
|
|
return this.resultData.detections
|
|
.filter( s => { return s.confidence >= this.detectorLevel})
|
|
.every( s => { return s.beenViewed })
|
|
} else {
|
|
return false
|
|
}
|
|
},
|
|
demoEnabled () {
|
|
return this.otherSettings.demo || this.demoMode
|
|
}
|
|
},
|
|
methods: {
|
|
chipGradient (confVal) {
|
|
let confFactor = confVal / 100
|
|
return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
|
|
},
|
|
async setData () {
|
|
if (this.reloadModel) {
|
|
await this.loadModel(this.modelLocation)
|
|
this.reloadModel = false
|
|
}
|
|
if (this.serverSettings && this.serverSettings.use) {
|
|
this.remoteDetect()
|
|
} else {
|
|
this.localDetect(this.imageView).then(dets => {
|
|
this.detecting = false
|
|
this.resultData = dets
|
|
this.uploadDirty = true
|
|
}).catch((e) => {
|
|
console.log(e.message)
|
|
this.detecting = false
|
|
this.resultData = {}
|
|
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
|
|
})
|
|
}
|
|
},
|
|
selectAll (ev) {
|
|
if (ev.target.checked) {
|
|
this.detectorLabels.forEach( s => s.detect = true )
|
|
} else {
|
|
this.detectorLabels.forEach( s => s.detect = false )
|
|
}
|
|
},
|
|
async selectImage (mode) {
|
|
this.imageLoadMode = mode
|
|
if (this.isCordova && mode == "camera") {
|
|
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
|
|
return
|
|
}
|
|
if (mode == "camera") {
|
|
this.videoAvailable = await this.openCamera(this.$refs.image_container)
|
|
if (this.videoAvailable) {
|
|
this.imageLoaded = false
|
|
this.imageView = null
|
|
this.$refs.image_cvs.style['background-image'] = 'none'
|
|
this.resultData = {}
|
|
var trackDetails = this.cameraStream.getVideoTracks()[0].getSettings()
|
|
var vidElement = this.$refs.vid_viewer
|
|
vidElement.width = trackDetails.width
|
|
vidElement.height = trackDetails.height
|
|
if (!this.otherSettings.disableVideo) {
|
|
this.videoFrameDetect(vidElement)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
if (mode == 'sample') {
|
|
f7.dialog.create({
|
|
title: 'Sample images',
|
|
buttons: [
|
|
{text: 'Sample 1', close: true, onClick: () => this.getImage('sample1'), cssClass: 'avn-dialog-button'},
|
|
{text: 'Sample 2', close: true, onClick: () => this.getImage('sample2'), cssClass: 'avn-dialog-button'},
|
|
{text: 'Sample 3', close: true, onClick: () => this.getImage('sample3'), cssClass: 'avn-dialog-button'},
|
|
{text: 'Cancel', close: true, color: 'red', cssClass: 'avn-dialog-button'}
|
|
],
|
|
verticalButtons: true
|
|
}).open()
|
|
return
|
|
}
|
|
this.$refs.image_chooser.click()
|
|
},
|
|
onFail (message) {
|
|
alert(`Camera fail: ${message}`)
|
|
},
|
|
selectChip ( iChip ) {
|
|
const [imCanvas, imageCtx] = this.resetView()
|
|
|
|
if (this.selectedChip == iChip) {
|
|
this.selectedChip = -1
|
|
return
|
|
}
|
|
|
|
if (iChip == 'redraw') {
|
|
if (this.selectedChip == -1) return
|
|
iChip = this.selectedChip
|
|
}
|
|
|
|
const boxCoords = this.box2cvs(this.resultData.detections[iChip])[0]
|
|
|
|
var boxLeft = boxCoords.cvsLeft
|
|
var boxTop = boxCoords.cvsTop
|
|
var boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
|
|
var boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
|
|
imageCtx.strokeRect(boxLeft,boxTop,boxWidth,boxHeight)
|
|
this.selectedChip = iChip
|
|
this.resultData.detections[iChip].beenViewed = true
|
|
},
|
|
deleteChip ( iChip ) {
|
|
f7.dialog.confirm(`${this.resultData.detections[iChip].label} is identified with ${this.resultData.detections[iChip].confidence.toFixed(1)}% confidence. Are you sure you want to delete it?`, () => {
|
|
this.resetView()
|
|
this.resultData.detections.splice(iChip, 1)
|
|
this.selectedChip = -1
|
|
this.uploadDirty = true
|
|
});
|
|
},
|
|
resetView () {
|
|
const imCanvas = this.$refs.image_cvs
|
|
const imageCtx = imCanvas.getContext("2d")
|
|
imCanvas.width = imCanvas.clientWidth
|
|
imCanvas.height = imCanvas.clientHeight
|
|
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
|
imageCtx.strokeStyle = 'yellow'
|
|
imageCtx.lineWidth = 3
|
|
return [imCanvas, imageCtx]
|
|
},
|
|
getImage (searchImage) {
|
|
let loadImage = new Promise(resolve => {
|
|
if (this.videoAvailable) {
|
|
this.closeCamera()
|
|
this.detecting = true
|
|
this.reloadModel = true
|
|
resolve(searchImage)
|
|
} else if (this.isCordova && this.imageLoadMode == "camera") {
|
|
this.detecting = true
|
|
resolve('data:image/jpg;base64,' + searchImage)
|
|
}
|
|
var reader = new FileReader()
|
|
reader.addEventListener("load", () => {
|
|
this.detecting = true
|
|
resolve(reader.result)
|
|
})
|
|
if (this.imageLoadMode == 'sample') {
|
|
fetch(`${this.isCordova ? 'https://localhost' : '.'}/samples/${this.detectorName}-${searchImage}.jpeg`).then( resp => {
|
|
return resp.blob()
|
|
}).then(respBlob => {
|
|
reader.readAsDataURL(respBlob)
|
|
}).catch((e) => {
|
|
console.log(e.message)
|
|
f7.dialog.alert(`Error loading image: ${e.message}`)
|
|
})
|
|
} else {
|
|
const fileImage = this.$refs.image_chooser.files[0]
|
|
this.$refs.image_chooser.value=[]
|
|
reader.readAsDataURL(fileImage)
|
|
}
|
|
})
|
|
loadImage.then((imgData) => {
|
|
this.imageLoaded = true
|
|
this.resultData = {}
|
|
this.selectedChip = -1
|
|
this.imageView = new Image()
|
|
this.imageView.src = imgData
|
|
return(this.imageView.decode())
|
|
}).then( () => {
|
|
const [imCanvas, _] = this.resetView()
|
|
imCanvas.style['background-image'] = `url(${this.imageView.src})`
|
|
/******
|
|
* setTimeout is not a good solution, but it's the only way
|
|
* I can find to not cut off drawing of the canvas background
|
|
******/
|
|
setTimeout(() => {
|
|
this.setData()
|
|
}, 1)
|
|
}).catch((e) => {
|
|
console.log(e.message)
|
|
f7.dialog.alert(`Error loading image: ${e.message}`)
|
|
})
|
|
},
|
|
async submitData () {
|
|
var uploadData = this.showResults
|
|
.filter( d => { return d.aboveThreshold && d.isSearched && !d.isDeleted })
|
|
.map( r => { return {"top": r.top, "left": r.left, "bottom": r.bottom, "right": r.right, "label": r.label}})
|
|
this.uploadUid = await this.uploadData(this.imageView.src.split(',')[1],uploadData,this.uploadUid)
|
|
if (this.uploadUid) { this.uploadDirty = false }
|
|
},
|
|
onLevelChange(value) {
|
|
this.detectorLevel = value
|
|
},
|
|
structureClick(e) {
|
|
const boxCoords = this.box2cvs(this.showResults)
|
|
var findBox = boxCoords.findIndex( (r, i) => { return r.cvsLeft <= e.offsetX &&
|
|
r.cvsRight >= e.offsetX &&
|
|
r.cvsTop <= e.offsetY &&
|
|
r.cvsBottom >= e.offsetY &&
|
|
this.resultData.detections[i].resultIndex > this.selectedChip &&
|
|
this.resultData.detections[i].aboveThreshold &&
|
|
this.resultData.detections[i].isSearched &&
|
|
!this.resultData.detections[i].isDeleted
|
|
})
|
|
this.selectChip(findBox >= 0 ? this.resultData.detections[findBox].resultIndex : this.selectedChip)
|
|
},
|
|
box2cvs(boxInput) {
|
|
if (!boxInput || boxInput.length == 0) return []
|
|
const boxList = boxInput.length ? boxInput : [boxInput]
|
|
const [imCanvas, imageCtx] = this.resetView()
|
|
var imgWidth
|
|
var imgHeight
|
|
const imgAspect = this.imageView.width / this.imageView.height
|
|
const rendAspect = imCanvas.width / imCanvas.height
|
|
if (imgAspect >= rendAspect) {
|
|
imgWidth = imCanvas.width
|
|
imgHeight = imCanvas.width / imgAspect
|
|
} else {
|
|
imgWidth = imCanvas.height * imgAspect
|
|
imgHeight = imCanvas.height
|
|
}
|
|
const cvsCoords = boxList.map( (d, i) => {
|
|
return {
|
|
"cvsLeft": (imCanvas.width - imgWidth) / 2 + d.left * imgWidth,
|
|
"cvsRight": (imCanvas.width - imgWidth) / 2 + d.right * imgWidth,
|
|
"cvsTop": (imCanvas.height - imgHeight) / 2 + d.top * imgHeight,
|
|
"cvsBottom": (imCanvas.height - imgHeight) / 2 + d.bottom * imgHeight
|
|
}
|
|
})
|
|
return cvsCoords
|
|
},
|
|
toggleFullscreen() {
|
|
if (document.fullscreenElement) {
|
|
document.exitFullscreen().then( () => {
|
|
this.isFullscreen = false
|
|
})
|
|
} else {
|
|
app.requestFullscreen().then( () => {
|
|
this.isFullscreen = true
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
</script> |