Compare commits
4 Commits
v0.5.0-alp
...
205f2da87d
| Author | SHA1 | Date | |
|---|---|---|---|
| 205f2da87d | |||
| 77b108af6c | |||
| cd6cad9e73 | |||
| c7622c3602 |
@@ -1,5 +1,5 @@
|
|||||||
<?xml version='1.0' encoding='utf-8'?>
|
<?xml version='1.0' encoding='utf-8'?>
|
||||||
<widget id="edu.midwestern.alvinn" version="0.5.0-alpha" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
|
<widget id="edu.midwestern.alvinn" version="0.5.0-rc" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
|
||||||
<name>ALVINN</name>
|
<name>ALVINN</name>
|
||||||
<description>Anatomy Lab Visual Identification Neural Network.</description>
|
<description>Anatomy Lab Visual Identification Neural Network.</description>
|
||||||
<author email="jgeorg@midwestern.edu" href="https://midwestern.edu">
|
<author email="jgeorg@midwestern.edu" href="https://midwestern.edu">
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "edu.midwestern.alvinn",
|
"name": "edu.midwestern.alvinn",
|
||||||
"displayName": "ALVINN",
|
"displayName": "ALVINN",
|
||||||
"version": "0.5.0-alpha",
|
"version": "0.5.0-rc",
|
||||||
"description": "Anatomy Lab Visual Identification Neural Network.",
|
"description": "Anatomy Lab Visual Identification Neural Network.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "alvinn",
|
"name": "alvinn",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.5.0-alpha",
|
"version": "0.5.0-rc",
|
||||||
"description": "ALVINN",
|
"description": "ALVINN",
|
||||||
"repository": "",
|
"repository": "",
|
||||||
"license": "UNLICENSED",
|
"license": "UNLICENSED",
|
||||||
@@ -14,8 +14,7 @@
|
|||||||
"cordova-ios": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run ios",
|
"cordova-ios": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run ios",
|
||||||
"build-cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova build android",
|
"build-cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova build android",
|
||||||
"cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run android",
|
"cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run android",
|
||||||
"postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/",
|
"postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/"
|
||||||
"preview": "vite preview"
|
|
||||||
},
|
},
|
||||||
"browserslist": [
|
"browserslist": [
|
||||||
"IOS >= 15",
|
"IOS >= 15",
|
||||||
|
|||||||
@@ -1,39 +1,45 @@
|
|||||||
import * as tf from '@tensorflow/tfjs'
|
import * as tf from '@tensorflow/tfjs'
|
||||||
|
import { f7 } from 'framework7-vue'
|
||||||
|
|
||||||
let model = null
|
let model = null
|
||||||
|
|
||||||
onmessage = function (e) {
|
self.onconnect = (e) => {
|
||||||
|
const port = e.ports[0];
|
||||||
|
|
||||||
|
port.onmessage = function (e) {
|
||||||
switch (e.data.call) {
|
switch (e.data.call) {
|
||||||
case 'loadModel':
|
case 'loadModel':
|
||||||
loadModel(e.data.weights,e.data.preload).then(() => {
|
loadModel('.' + e.data.weights,e.data.preload).then(() => {
|
||||||
postMessage({success: 'model'})
|
port.postMessage({success: 'model'})
|
||||||
}).catch((err) => {
|
}).catch((err) => {
|
||||||
postMessage({error: true, message: err.message})
|
port.postMessage({error: true, message: err.message})
|
||||||
})
|
})
|
||||||
break
|
break
|
||||||
case 'localDetect':
|
case 'localDetect':
|
||||||
localDetect(e.data.image).then((dets) => {
|
localDetect(e.data.image).then((dets) => {
|
||||||
postMessage({success: 'detection', detections: dets})
|
port.postMessage({success: 'detection', detections: dets})
|
||||||
}).catch((err) => {
|
}).catch((err) => {
|
||||||
//throw (err)
|
port.postMessage({error: true, message: err.message})
|
||||||
postMessage({error: true, message: err.message})
|
|
||||||
})
|
})
|
||||||
e.data.image.close()
|
e.data.image.close()
|
||||||
break
|
break
|
||||||
case 'videoFrame':
|
case 'videoFrame':
|
||||||
videoFrame(e.data.image).then((frameDet) =>{
|
videoFrame(e.data.image).then((frameDet) =>{
|
||||||
postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH})
|
port.postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH})
|
||||||
}).catch((err) => {
|
}).catch((err) => {
|
||||||
postMessage({error: true, message: err.message})
|
port.postMessage({error: true, message: err.message})
|
||||||
})
|
})
|
||||||
e.data.image.close()
|
e.data.image.close()
|
||||||
break
|
break
|
||||||
default:
|
default:
|
||||||
console.log('Worker message incoming:')
|
console.log('Worker message incoming:')
|
||||||
console.log(e)
|
console.log(e)
|
||||||
postMessage({result1: 'First result', result2: 'Second result'})
|
port.postMessage({result1: 'First result', result2: 'Second result'})
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
port.start()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadModel(weights, preload) {
|
async function loadModel(weights, preload) {
|
||||||
@@ -57,7 +63,7 @@ async function loadModel(weights, preload) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function localDetect(imageData) {
|
async function localDetect(imageData) {
|
||||||
console.time('sw: pre-process')
|
console.time('pre-process')
|
||||||
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
||||||
let gTense = null
|
let gTense = null
|
||||||
const input = tf.tidy(() => {
|
const input = tf.tidy(() => {
|
||||||
@@ -65,15 +71,15 @@ async function localDetect(imageData) {
|
|||||||
return tf.concat([gTense,gTense,gTense],3)
|
return tf.concat([gTense,gTense,gTense],3)
|
||||||
})
|
})
|
||||||
tf.dispose(gTense)
|
tf.dispose(gTense)
|
||||||
console.timeEnd('sw: pre-process')
|
console.timeEnd('pre-process')
|
||||||
|
|
||||||
console.time('sw: run prediction')
|
console.time('run prediction')
|
||||||
const res = model.predict(input)
|
const res = model.predict(input)
|
||||||
const tRes = tf.transpose(res,[0,2,1])
|
const tRes = tf.transpose(res,[0,2,1])
|
||||||
const rawRes = tRes.arraySync()[0]
|
const rawRes = tRes.arraySync()[0]
|
||||||
console.timeEnd('sw: run prediction')
|
console.timeEnd('run prediction')
|
||||||
|
|
||||||
console.time('sw: post-process')
|
console.time('post-process')
|
||||||
const outputSize = res.shape[1]
|
const outputSize = res.shape[1]
|
||||||
let rawBoxes = []
|
let rawBoxes = []
|
||||||
let rawScores = []
|
let rawScores = []
|
||||||
@@ -138,14 +144,14 @@ async function localDetect(imageData) {
|
|||||||
}
|
}
|
||||||
tf.dispose(res)
|
tf.dispose(res)
|
||||||
tf.dispose(input)
|
tf.dispose(input)
|
||||||
console.timeEnd('sw: post-process')
|
console.timeEnd('post-process')
|
||||||
|
|
||||||
return output || { detections: [] }
|
return output || { detections: [] }
|
||||||
}
|
}
|
||||||
|
|
||||||
async function videoFrame (vidData) {
|
async function videoFrame (vidData) {
|
||||||
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
||||||
console.time('sw: frame-process')
|
console.time('frame-process')
|
||||||
let rawCoords = []
|
let rawCoords = []
|
||||||
try {
|
try {
|
||||||
const input = tf.tidy(() => {
|
const input = tf.tidy(() => {
|
||||||
@@ -171,6 +177,6 @@ async function videoFrame (vidData) {
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.log(e)
|
console.log(e)
|
||||||
}
|
}
|
||||||
console.timeEnd('sw: frame-process')
|
console.timeEnd('frame-process')
|
||||||
return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
|
return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
|
||||||
}
|
}
|
||||||
@@ -79,7 +79,6 @@
|
|||||||
.then((mod) => { return mod.text() })
|
.then((mod) => { return mod.text() })
|
||||||
this.siteConf = YAML.parse(confText)
|
this.siteConf = YAML.parse(confText)
|
||||||
}
|
}
|
||||||
if (window.safari !== undefined) {store().safariDetected()}
|
|
||||||
const loadSiteSettings = localStorage.getItem('siteSettings')
|
const loadSiteSettings = localStorage.getItem('siteSettings')
|
||||||
if (loadSiteSettings) {
|
if (loadSiteSettings) {
|
||||||
let loadedSettings = JSON.parse(loadSiteSettings)
|
let loadedSettings = JSON.parse(loadSiteSettings)
|
||||||
|
|||||||
@@ -4,13 +4,12 @@ const state = reactive({
|
|||||||
disclaimerAgreement: false,
|
disclaimerAgreement: false,
|
||||||
enabledRegions: ['thorax','abdomen','limbs','head'],
|
enabledRegions: ['thorax','abdomen','limbs','head'],
|
||||||
regionIconSet: Math.floor(Math.random() * 3) + 1,
|
regionIconSet: Math.floor(Math.random() * 3) + 1,
|
||||||
version: '0.5.0-alpha',
|
version: '0.5.0-rc',
|
||||||
fullscreen: false,
|
fullscreen: false,
|
||||||
useExternal: 'optional',
|
useExternal: 'optional',
|
||||||
siteDemo: false,
|
siteDemo: false,
|
||||||
externalServerList: [],
|
externalServerList: [],
|
||||||
infoUrl: false,
|
infoUrl: false
|
||||||
safariBrowser: false
|
|
||||||
})
|
})
|
||||||
|
|
||||||
const set = (config, confObj) => {
|
const set = (config, confObj) => {
|
||||||
@@ -22,10 +21,6 @@ const agree = () => {
|
|||||||
state.disclaimerAgreement = true
|
state.disclaimerAgreement = true
|
||||||
}
|
}
|
||||||
|
|
||||||
const safariDetected = () => {
|
|
||||||
state.safariBrowser = true
|
|
||||||
}
|
|
||||||
|
|
||||||
const getServerList = () => {
|
const getServerList = () => {
|
||||||
if (state.useExternal == 'required') {
|
if (state.useExternal == 'required') {
|
||||||
return state.externalServerList[0]
|
return state.externalServerList[0]
|
||||||
@@ -55,10 +50,8 @@ export default () => ({
|
|||||||
getVersion: computed(() => state.version),
|
getVersion: computed(() => state.version),
|
||||||
getIconSet: computed(() => state.regionIconSet),
|
getIconSet: computed(() => state.regionIconSet),
|
||||||
getInfoUrl: computed(() => state.infoUrl),
|
getInfoUrl: computed(() => state.infoUrl),
|
||||||
isSafari: computed(() => state.safariBrowser),
|
|
||||||
set,
|
set,
|
||||||
agree,
|
agree,
|
||||||
safariDetected,
|
|
||||||
getServerList,
|
getServerList,
|
||||||
toggleFullscreen
|
toggleFullscreen
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -41,20 +41,16 @@ export default {
|
|||||||
tempCtx.drawImage(vidViewer, 0, 0)
|
tempCtx.drawImage(vidViewer, 0, 0)
|
||||||
this.getImage(tempCVS.toDataURL())
|
this.getImage(tempCVS.toDataURL())
|
||||||
},
|
},
|
||||||
async videoFrameDetectWorker (vidData) {
|
async videoFrameDetect (vidData) {
|
||||||
const startDetection = () => {
|
const vidWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
|
||||||
createImageBitmap(vidData).then(imVideoFrame => {
|
vidWorker.port.onmessage = (eVid) => {
|
||||||
this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
|
self = this
|
||||||
})
|
|
||||||
}
|
|
||||||
vidData.addEventListener('resize',startDetection,{once: true})
|
|
||||||
this.vidWorker.onmessage = (eVid) => {
|
|
||||||
if (eVid.data.error) {
|
if (eVid.data.error) {
|
||||||
console.log(eVid.data.message)
|
console.log(eVid.data.message)
|
||||||
f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`)
|
f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`)
|
||||||
} else if (this.videoAvailable) {
|
} else if (this.videoAvailable) {
|
||||||
createImageBitmap(vidData).then(imVideoFrame => {
|
createImageBitmap(vidData).then(imVideoFrame => {
|
||||||
this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
|
vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
|
||||||
})
|
})
|
||||||
if (eVid.data.coords) {
|
if (eVid.data.coords) {
|
||||||
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
||||||
@@ -69,6 +65,7 @@ export default {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
|
||||||
const imCanvas = this.$refs.image_cvs
|
const imCanvas = this.$refs.image_cvs
|
||||||
const imageCtx = imCanvas.getContext("2d")
|
const imageCtx = imCanvas.getContext("2d")
|
||||||
const target = this.$refs.target_image
|
const target = this.$refs.target_image
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
</f7-navbar>
|
</f7-navbar>
|
||||||
<f7-block class="detect-grid">
|
<f7-block class="detect-grid">
|
||||||
<div class="image-container" ref="image_container">
|
<div class="image-container" ref="image_container">
|
||||||
<SvgIcon v-if="!imageView.src && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)"/>
|
<SvgIcon v-if="!imageView.src && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)" @click="selectImage" />
|
||||||
<div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
|
<div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
|
||||||
<video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
|
<video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
|
||||||
<f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
|
<f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
|
||||||
@@ -140,8 +140,6 @@
|
|||||||
import detectionMixin from './detection-mixin'
|
import detectionMixin from './detection-mixin'
|
||||||
import cameraMixin from './camera-mixin'
|
import cameraMixin from './camera-mixin'
|
||||||
|
|
||||||
import detectionWorker from '@/assets/detect-worker.js?worker&inline'
|
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
mixins: [submitMixin, detectionMixin, cameraMixin],
|
mixins: [submitMixin, detectionMixin, cameraMixin],
|
||||||
props: {
|
props: {
|
||||||
@@ -180,8 +178,7 @@
|
|||||||
videoAvailable: false,
|
videoAvailable: false,
|
||||||
cameraStream: null,
|
cameraStream: null,
|
||||||
infoLinkPos: {},
|
infoLinkPos: {},
|
||||||
detectWorker: null,
|
workerScript: null
|
||||||
vidWorker: null
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
setup() {
|
setup() {
|
||||||
@@ -190,6 +187,7 @@
|
|||||||
created () {
|
created () {
|
||||||
let loadOtherSettings = localStorage.getItem('otherSettings')
|
let loadOtherSettings = localStorage.getItem('otherSettings')
|
||||||
if (loadOtherSettings) this.otherSettings = JSON.parse(loadOtherSettings)
|
if (loadOtherSettings) this.otherSettings = JSON.parse(loadOtherSettings)
|
||||||
|
let modelRoot = this.isCordova ? 'https://localhost' : '.'
|
||||||
this.detectorName = this.f7route.params.region
|
this.detectorName = this.f7route.params.region
|
||||||
switch (this.detectorName) {
|
switch (this.detectorName) {
|
||||||
case 'thorax':
|
case 'thorax':
|
||||||
@@ -205,9 +203,9 @@
|
|||||||
this.activeRegion = 3
|
this.activeRegion = 3
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
this.modelLocation = URL.parse(`../models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`,import.meta.url).href
|
this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
|
||||||
this.miniLocation = URL.parse(`../models/${this.detectorName}-mini/model.json`,import.meta.url).href
|
this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
|
||||||
fetch(URL.parse(`../models/${this.detectorName}/classes.json`,import.meta.url).href)
|
fetch(`${modelRoot}/models/${this.detectorName}/classes.json`)
|
||||||
.then((mod) => { return mod.json() })
|
.then((mod) => { return mod.json() })
|
||||||
.then((classes) => {
|
.then((classes) => {
|
||||||
this.classesList = classes
|
this.classesList = classes
|
||||||
@@ -217,8 +215,8 @@
|
|||||||
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
|
if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
|
||||||
},
|
},
|
||||||
mounted () {
|
mounted () {
|
||||||
this.detectWorker = new detectionWorker()
|
const mountWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
|
||||||
this.detectWorker.onmessage = (eMount) => {
|
mountWorker.port.onmessage = (eMount) => {
|
||||||
self = this
|
self = this
|
||||||
if (eMount.data.error) {
|
if (eMount.data.error) {
|
||||||
console.log(eMount.data.message)
|
console.log(eMount.data.message)
|
||||||
@@ -226,32 +224,13 @@
|
|||||||
}
|
}
|
||||||
self.modelLoading = false
|
self.modelLoading = false
|
||||||
}
|
}
|
||||||
this.vidWorker = new detectionWorker()
|
|
||||||
this.vidWorker.onmessage = (eMount) => {
|
|
||||||
self = this
|
|
||||||
if (eMount.data.error) {
|
|
||||||
console.log(eMount.data.message)
|
|
||||||
f7.dialog.alert(`ALVINN AI nano model error: ${eMount.data.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.serverSettings && this.serverSettings.use) {
|
if (this.serverSettings && this.serverSettings.use) {
|
||||||
this.getRemoteLabels()
|
this.getRemoteLabels()
|
||||||
this.modelLoading = false
|
this.modelLoading = false
|
||||||
} else {
|
} else {
|
||||||
this.modelLoading = true
|
this.modelLoading = true
|
||||||
if (this.isSafari) {
|
mountWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true})
|
||||||
this.loadModel(this.modelLocation, true).then(() => {
|
|
||||||
this.modelLoading = false
|
|
||||||
}).catch((e) => {
|
|
||||||
console.log(e.message)
|
|
||||||
f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
|
|
||||||
this.modelLoading = false
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true})
|
|
||||||
this.vidWorker.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') }
|
window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') }
|
||||||
},
|
},
|
||||||
@@ -304,7 +283,7 @@
|
|||||||
infoLinkTarget () {
|
infoLinkTarget () {
|
||||||
if (!this.getInfoUrl) return ''
|
if (!this.getInfoUrl) return ''
|
||||||
let structure = this.showResults.find( r => r.resultIndex == this.selectedChip)
|
let structure = this.showResults.find( r => r.resultIndex == this.selectedChip)
|
||||||
return structure ? this.getInfoUrl + structure.label.replaceAll(' ','_') : ''
|
return this.getInfoUrl + structure.label.replaceAll(' ','_')
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
methods: {
|
methods: {
|
||||||
@@ -313,12 +292,12 @@
|
|||||||
return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
|
return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
|
||||||
},
|
},
|
||||||
async setData () {
|
async setData () {
|
||||||
this.detectWorker.onmessage = (eDetect) => {
|
const detectWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
|
||||||
|
detectWorker.port.onmessage = (eDetect) => {
|
||||||
self = this
|
self = this
|
||||||
if (eDetect.data.error) {
|
if (eDetect.data.error) {
|
||||||
self.detecting = false
|
self.detecting = false
|
||||||
self.resultData = {}
|
self.resultData = {}
|
||||||
loadFailure()
|
|
||||||
f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
|
f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
|
||||||
} else if (eDetect.data.success == 'detection') {
|
} else if (eDetect.data.success == 'detection') {
|
||||||
self.detecting = false
|
self.detecting = false
|
||||||
@@ -328,45 +307,28 @@
|
|||||||
}
|
}
|
||||||
self.uploadDirty = true
|
self.uploadDirty = true
|
||||||
} else if (eDetect.data.success == 'model') {
|
} else if (eDetect.data.success == 'model') {
|
||||||
self.reloadModel = false
|
this.reloadModel = false
|
||||||
loadSuccess()
|
loadSuccess(true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let loadSuccess = null
|
let loadSuccess = null
|
||||||
let loadFailure = null
|
let loadFailure = null
|
||||||
let modelReloading = null
|
let modelReloading = new Promise((res, rej) => {
|
||||||
if (this.isSafari && this.reloadModel) {
|
|
||||||
await this.loadModel(this.modelLocation)
|
|
||||||
this.reloadModel = false
|
|
||||||
} else {
|
|
||||||
modelReloading = new Promise((res, rej) => {
|
|
||||||
loadSuccess = res
|
loadSuccess = res
|
||||||
loadFailure = rej
|
loadFailure = rej
|
||||||
if (this.reloadModel) {
|
if (this.reloadModel) {
|
||||||
this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
|
detectWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation})
|
||||||
} else {
|
} else {
|
||||||
loadSuccess()
|
loadSuccess(true)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
if (this.serverSettings && this.serverSettings.use) {
|
if (this.serverSettings && this.serverSettings.use) {
|
||||||
this.remoteDetect()
|
this.remoteDetect()
|
||||||
} else if (!this.isSafari) {
|
|
||||||
Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
|
|
||||||
this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
|
|
||||||
})
|
|
||||||
} else {
|
} else {
|
||||||
this.localDetect(this.imageView).then(dets => {
|
Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
|
||||||
this.detecting = false
|
detectWorker.port.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
|
||||||
this.resultData = dets
|
|
||||||
this.uploadDirty = true
|
|
||||||
}).catch((e) => {
|
|
||||||
console.log(e.message)
|
|
||||||
this.detecting = false
|
|
||||||
this.resultData = {}
|
|
||||||
f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -383,10 +345,9 @@
|
|||||||
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
|
navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (mode == "camera" && !this.otherSettings.disableVideo) {
|
if (mode == "camera") {
|
||||||
this.videoAvailable = await this.openCamera(this.$refs.image_container)
|
this.videoAvailable = await this.openCamera(this.$refs.image_container)
|
||||||
if (this.videoAvailable) {
|
if (this.videoAvailable) {
|
||||||
this.selectedChip = -1
|
|
||||||
this.imageLoaded = false
|
this.imageLoaded = false
|
||||||
this.imageView.src = null
|
this.imageView.src = null
|
||||||
this.$refs.image_cvs.style['background-image'] = 'none'
|
this.$refs.image_cvs.style['background-image'] = 'none'
|
||||||
@@ -395,10 +356,8 @@
|
|||||||
var vidElement = this.$refs.vid_viewer
|
var vidElement = this.$refs.vid_viewer
|
||||||
vidElement.width = trackDetails.width
|
vidElement.width = trackDetails.width
|
||||||
vidElement.height = trackDetails.height
|
vidElement.height = trackDetails.height
|
||||||
if (this.isSafari) {
|
if (!this.otherSettings.disableVideo) {
|
||||||
this.videoFrameDetect(vidElement)
|
this.videoFrameDetect(vidElement)
|
||||||
} else {
|
|
||||||
this.videoFrameDetectWorker(vidElement)
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -512,9 +471,13 @@
|
|||||||
}).then( () => {
|
}).then( () => {
|
||||||
const [imCanvas, _] = this.resetView()
|
const [imCanvas, _] = this.resetView()
|
||||||
imCanvas.style['background-image'] = `url(${this.imageView.src})`
|
imCanvas.style['background-image'] = `url(${this.imageView.src})`
|
||||||
f7.utils.nextFrame(() => {
|
/******
|
||||||
|
* setTimeout is not a good solution, but it's the only way
|
||||||
|
* I can find to not cut off drawing of the canvas background
|
||||||
|
******/
|
||||||
|
// setTimeout(() => {
|
||||||
this.setData()
|
this.setData()
|
||||||
})
|
// }, 1)
|
||||||
}).catch((e) => {
|
}).catch((e) => {
|
||||||
console.log(e.message)
|
console.log(e.message)
|
||||||
f7.dialog.alert(`Error loading image: ${e.message}`)
|
f7.dialog.alert(`Error loading image: ${e.message}`)
|
||||||
|
|||||||
@@ -1,114 +1,7 @@
|
|||||||
import * as tf from '@tensorflow/tfjs'
|
|
||||||
import { f7 } from 'framework7-vue'
|
import { f7 } from 'framework7-vue'
|
||||||
|
|
||||||
let model = null
|
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
methods: {
|
methods: {
|
||||||
async loadModel(weights, preload) {
|
|
||||||
if (model && model.modelURL == weights) {
|
|
||||||
return model
|
|
||||||
} else if (model) {
|
|
||||||
tf.dispose(model)
|
|
||||||
}
|
|
||||||
model = await tf.loadGraphModel(weights)
|
|
||||||
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
||||||
/*****************
|
|
||||||
* If preloading then run model
|
|
||||||
* once on fake data to preload
|
|
||||||
* weights for a faster response
|
|
||||||
*****************/
|
|
||||||
if (preload) {
|
|
||||||
const dummyT = tf.ones([1,modelWidth,modelHeight,3])
|
|
||||||
model.predict(dummyT)
|
|
||||||
}
|
|
||||||
return model
|
|
||||||
},
|
|
||||||
async localDetect(imageData) {
|
|
||||||
console.time('mx: pre-process')
|
|
||||||
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
||||||
let gTense = null
|
|
||||||
const input = tf.tidy(() => {
|
|
||||||
gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
|
|
||||||
return tf.concat([gTense,gTense,gTense],3)
|
|
||||||
})
|
|
||||||
tf.dispose(gTense)
|
|
||||||
console.timeEnd('mx: pre-process')
|
|
||||||
|
|
||||||
console.time('mx: run prediction')
|
|
||||||
const res = model.predict(input)
|
|
||||||
const tRes = tf.transpose(res,[0,2,1])
|
|
||||||
const rawRes = tRes.arraySync()[0]
|
|
||||||
console.timeEnd('mx: run prediction')
|
|
||||||
|
|
||||||
console.time('mx: post-process')
|
|
||||||
const outputSize = res.shape[1]
|
|
||||||
let rawBoxes = []
|
|
||||||
let rawScores = []
|
|
||||||
|
|
||||||
for (var i = 0; i < rawRes.length; i++) {
|
|
||||||
var getScores = rawRes[i].slice(4)
|
|
||||||
if (getScores.every( s => s < .05)) { continue }
|
|
||||||
var getBox = rawRes[i].slice(0,4)
|
|
||||||
var boxCalc = [
|
|
||||||
(getBox[0] - (getBox[2] / 2)) / modelWidth,
|
|
||||||
(getBox[1] - (getBox[3] / 2)) / modelHeight,
|
|
||||||
(getBox[0] + (getBox[2] / 2)) / modelWidth,
|
|
||||||
(getBox[1] + (getBox[3] / 2)) / modelHeight,
|
|
||||||
]
|
|
||||||
rawBoxes.push(boxCalc)
|
|
||||||
rawScores.push(getScores)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (rawBoxes.length > 0) {
|
|
||||||
const tBoxes = tf.tensor2d(rawBoxes)
|
|
||||||
let tScores = null
|
|
||||||
let resBoxes = null
|
|
||||||
let validBoxes = []
|
|
||||||
let structureScores = null
|
|
||||||
let boxes_data = []
|
|
||||||
let scores_data = []
|
|
||||||
let classes_data = []
|
|
||||||
for (var c = 0; c < outputSize - 4; c++) {
|
|
||||||
structureScores = rawScores.map(x => x[c])
|
|
||||||
tScores = tf.tensor1d(structureScores)
|
|
||||||
resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
|
|
||||||
validBoxes = resBoxes.dataSync()
|
|
||||||
tf.dispose(resBoxes)
|
|
||||||
if (validBoxes) {
|
|
||||||
boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
|
|
||||||
var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
|
|
||||||
scores_data.push(...outputScores)
|
|
||||||
classes_data.push(...outputScores.fill(c))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
validBoxes = []
|
|
||||||
tf.dispose(tBoxes)
|
|
||||||
tf.dispose(tScores)
|
|
||||||
tf.dispose(tRes)
|
|
||||||
const valid_detections_data = classes_data.length
|
|
||||||
var output = {
|
|
||||||
detections: []
|
|
||||||
}
|
|
||||||
for (var i =0; i < valid_detections_data; i++) {
|
|
||||||
var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
|
|
||||||
output.detections.push({
|
|
||||||
"top": dTop,
|
|
||||||
"left": dLeft,
|
|
||||||
"bottom": dBottom,
|
|
||||||
"right": dRight,
|
|
||||||
"label": this.detectorLabels[classes_data[i]].name,
|
|
||||||
"confidence": scores_data[i] * 100
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tf.dispose(res)
|
|
||||||
tf.dispose(input)
|
|
||||||
console.timeEnd('mx: post-process')
|
|
||||||
|
|
||||||
return output || { detections: [] }
|
|
||||||
},
|
|
||||||
getRemoteLabels() {
|
getRemoteLabels() {
|
||||||
var self = this
|
var self = this
|
||||||
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
|
var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
|
||||||
@@ -172,65 +65,5 @@ export default {
|
|||||||
this.detecting = false
|
this.detecting = false
|
||||||
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
|
f7.dialog.alert('No connection to remote ALVINN instance. Please check app settings.')
|
||||||
},
|
},
|
||||||
async videoFrameDetect (vidData) {
|
|
||||||
await this.loadModel(this.miniLocation)
|
|
||||||
const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
|
|
||||||
const imCanvas = this.$refs.image_cvs
|
|
||||||
const imageCtx = imCanvas.getContext("2d")
|
|
||||||
const target = this.$refs.target_image
|
|
||||||
await tf.nextFrame();
|
|
||||||
imCanvas.width = imCanvas.clientWidth
|
|
||||||
imCanvas.height = imCanvas.clientHeight
|
|
||||||
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
|
||||||
var imgWidth
|
|
||||||
var imgHeight
|
|
||||||
const imgAspect = vidData.width / vidData.height
|
|
||||||
const rendAspect = imCanvas.width / imCanvas.height
|
|
||||||
if (imgAspect >= rendAspect) {
|
|
||||||
imgWidth = imCanvas.width
|
|
||||||
imgHeight = imCanvas.width / imgAspect
|
|
||||||
} else {
|
|
||||||
imgWidth = imCanvas.height * imgAspect
|
|
||||||
imgHeight = imCanvas.height
|
|
||||||
}
|
|
||||||
while (this.videoAvailable) {
|
|
||||||
console.time('mx: frame-process')
|
|
||||||
try {
|
|
||||||
const input = tf.tidy(() => {
|
|
||||||
return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
|
|
||||||
})
|
|
||||||
const res = model.predict(input)
|
|
||||||
const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
|
|
||||||
|
|
||||||
let rawCoords = []
|
|
||||||
if (rawRes) {
|
|
||||||
for (var i = 0; i < rawRes.length; i++) {
|
|
||||||
let getScores = rawRes[i].slice(4)
|
|
||||||
if (getScores.some( s => s > .5)) {
|
|
||||||
let foundTarget = rawRes[i].slice(0,2)
|
|
||||||
foundTarget.push(Math.max(...getScores))
|
|
||||||
rawCoords.push(foundTarget)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
|
|
||||||
for (var coord of rawCoords) {
|
|
||||||
console.log(`x: ${coord[0]}, y: ${coord[1]}`)
|
|
||||||
let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
|
|
||||||
let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
|
|
||||||
imageCtx.globalAlpha = coord[2]
|
|
||||||
imageCtx.drawImage(target, pointX, pointY, 20, 20)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tf.dispose(input)
|
|
||||||
tf.dispose(res)
|
|
||||||
tf.dispose(rawRes)
|
|
||||||
} catch (e) {
|
|
||||||
console.log(e)
|
|
||||||
}
|
|
||||||
console.timeEnd('mx: frame-process')
|
|
||||||
await tf.nextFrame();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,7 +8,6 @@
|
|||||||
<f7-block-title medium>Details</f7-block-title>
|
<f7-block-title medium>Details</f7-block-title>
|
||||||
<f7-list>
|
<f7-list>
|
||||||
<f7-list-item title="Version" :after="alvinnVersion"></f7-list-item>
|
<f7-list-item title="Version" :after="alvinnVersion"></f7-list-item>
|
||||||
<f7-list-item v-if="isSafari" title="Safari" after="Workers disabled"></f7-list-item>
|
|
||||||
</f7-list>
|
</f7-list>
|
||||||
<f7-block-title medium>Models</f7-block-title>
|
<f7-block-title medium>Models</f7-block-title>
|
||||||
<f7-list style="width: 100%;">
|
<f7-list style="width: 100%;">
|
||||||
@@ -53,7 +52,6 @@
|
|||||||
miniHeadneckDetails: {},
|
miniHeadneckDetails: {},
|
||||||
alvinnVersion: store().getVersion,
|
alvinnVersion: store().getVersion,
|
||||||
isCordova: !!window.cordova,
|
isCordova: !!window.cordova,
|
||||||
isSafari: store().isSafari,
|
|
||||||
otherSettings: {}
|
otherSettings: {}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user