Cleanup for 0.5.0 alpha release

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

cordova/config.xml

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<widget id="edu.midwestern.alvinn" version="0.5.0-rc" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
+<widget id="edu.midwestern.alvinn" version="0.5.0-alpha" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
     <name>ALVINN</name>
     <description>Anatomy Lab Visual Identification Neural Network.</description>
     <author email="jgeorg@midwestern.edu" href="https://midwestern.edu">

cordova/package.json

@@ -1,7 +1,7 @@
 {
   "name": "edu.midwestern.alvinn",
   "displayName": "ALVINN",
-  "version": "0.5.0-rc",
+  "version": "0.5.0-alpha",
   "description": "Anatomy Lab Visual Identification Neural Network.",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "alvinn",
   "private": true,
-  "version": "0.5.0-rc",
+  "version": "0.5.0-alpha",
   "description": "ALVINN",
   "repository": "",
   "license": "UNLICENSED",
@@ -14,7 +14,8 @@
     "cordova-ios": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run ios",
     "build-cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova build android",
     "cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run android",
-    "postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/"
+    "postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/",
+    "preview": "vite preview"
   },
   "browserslist": [
     "IOS >= 15",

src/assets/detect-worker.js

@@ -0,0 +1,176 @@
+import * as tf  from '@tensorflow/tfjs'
+
+let model = null
+
+onmessage = function (e) {
+  switch (e.data.call) {
+    case 'loadModel':
+      loadModel(e.data.weights,e.data.preload).then(() => {
+        postMessage({success: 'model'})
+      }).catch((err) => {
+        postMessage({error: true, message: err.message})
+      })
+      break
+    case 'localDetect':
+      localDetect(e.data.image).then((dets) => {
+        postMessage({success: 'detection', detections: dets})
+      }).catch((err) => {
+        //throw (err)
+        postMessage({error: true, message: err.message})
+      })
+      e.data.image.close()
+      break
+    case 'videoFrame':
+      videoFrame(e.data.image).then((frameDet) =>{
+        postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH})
+      }).catch((err) => {
+        postMessage({error: true, message: err.message})
+      })
+      e.data.image.close()
+      break
+    default:
+      console.log('Worker message incoming:')
+      console.log(e)
+      postMessage({result1: 'First result', result2: 'Second result'})
+      break
+  }
+}
+
+async function loadModel(weights, preload) {
+  if (model && model.modelURL == weights) {
+    return model
+  } else if (model) {
+    tf.dispose(model)
+  }
+  model = await tf.loadGraphModel(weights)
+  const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+  /*****************
+   * If preloading then run model
+   * once on fake data to preload
+   * weights for a faster response
+   *****************/
+  if (preload) {
+    const dummyT = tf.ones([1,modelWidth,modelHeight,3])
+    model.predict(dummyT)
+  }
+  return model
+}
+
+async function localDetect(imageData) {
+  console.time('sw: pre-process')
+  const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+  let gTense = null
+  const input = tf.tidy(() => {
+    gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
+    return tf.concat([gTense,gTense,gTense],3)
+  })
+  tf.dispose(gTense)
+  console.timeEnd('sw: pre-process')
+
+  console.time('sw: run prediction')
+  const res = model.predict(input)
+  const tRes = tf.transpose(res,[0,2,1])
+  const rawRes = tRes.arraySync()[0]
+  console.timeEnd('sw: run prediction')
+
+  console.time('sw: post-process')
+  const outputSize = res.shape[1]
+  let rawBoxes = []
+  let rawScores = []
+
+  for (var i = 0; i < rawRes.length; i++) {
+    var getScores = rawRes[i].slice(4)
+    if (getScores.every( s => s < .05)) { continue }
+    var getBox = rawRes[i].slice(0,4)
+    var boxCalc = [
+      (getBox[0] - (getBox[2] / 2)) / modelWidth,
+      (getBox[1] - (getBox[3] / 2)) / modelHeight,
+      (getBox[0] + (getBox[2] / 2)) / modelWidth,
+      (getBox[1] + (getBox[3] / 2)) / modelHeight,
+    ]
+    rawBoxes.push(boxCalc)
+    rawScores.push(getScores)
+  }
+
+  if (rawBoxes.length > 0) {
+    const tBoxes = tf.tensor2d(rawBoxes)
+    let tScores = null
+    let resBoxes = null
+    let validBoxes = []
+    let structureScores = null
+    let boxes_data = []
+    let scores_data = []
+    let classes_data = []
+    for (var c = 0; c < outputSize - 4; c++) {
+      structureScores = rawScores.map(x => x[c])
+      tScores = tf.tensor1d(structureScores)
+      resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
+      validBoxes = resBoxes.dataSync()
+      tf.dispose(resBoxes)
+      if (validBoxes) {
+        boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
+        var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
+        scores_data.push(...outputScores)
+        classes_data.push(...outputScores.fill(c))
+      }
+    }
+
+    validBoxes = []
+    tf.dispose(tBoxes)
+    tf.dispose(tScores)
+    tf.dispose(tRes)
+    const valid_detections_data = classes_data.length
+    var output = {
+      detections: []
+    }
+    for (var i =0; i < valid_detections_data; i++) {
+      var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
+      output.detections.push({
+        "top": dTop,
+        "left": dLeft,
+        "bottom": dBottom,
+        "right": dRight,
+//        "label": this.detectorLabels[classes_data[i]].name,
+        "label": classes_data[i],
+        "confidence": scores_data[i] * 100
+      })
+    }
+  }
+  tf.dispose(res)
+  tf.dispose(input)
+  console.timeEnd('sw: post-process')
+
+  return output || { detections: [] }
+}
+
+async function videoFrame (vidData) {
+  const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+  console.time('sw: frame-process')
+  let rawCoords = []
+  try {
+    const input = tf.tidy(() => {
+      return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
+    })
+    const res = model.predict(input)
+    const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
+
+    if (rawRes) {
+      for (var i = 0; i < rawRes.length; i++) {
+        let getScores = rawRes[i].slice(4) 
+        if (getScores.some( s => s > .5)) {
+          let foundTarget = rawRes[i].slice(0,2)
+          foundTarget.push(Math.max(...getScores))
+          rawCoords.push(foundTarget)
+        }
+      }
+
+    }
+    tf.dispose(input)
+    tf.dispose(res)
+    tf.dispose(rawRes)
+  } catch (e) {
+    console.log(e)
+  }
+  console.timeEnd('sw: frame-process')
+  return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
+}

src/components/app.vue

@@ -79,6 +79,7 @@
           .then((mod) => { return mod.text() })
         this.siteConf = YAML.parse(confText)
       }
+      if (window.safari !== undefined) {store().safariDetected()}
       const loadSiteSettings = localStorage.getItem('siteSettings')
       if (loadSiteSettings) {
         let loadedSettings = JSON.parse(loadSiteSettings)

src/js/store.js

@@ -4,12 +4,13 @@ const state = reactive({
     disclaimerAgreement: false,
     enabledRegions: ['thorax','abdomen','limbs','head'],
     regionIconSet: Math.floor(Math.random() * 3) + 1,
-    version: '0.5.0-rc',
+    version: '0.5.0-alpha',
     fullscreen: false,
     useExternal: 'optional',
     siteDemo: false,
     externalServerList: [],
-    infoUrl: false
+    infoUrl: false,
+    safariBrowser: false
   })
 
 const set = (config, confObj) => {
@@ -21,6 +22,10 @@ const agree = () => {
   state.disclaimerAgreement = true
 }
 
+const safariDetected = () => {
+  state.safariBrowser = true
+}
+
 const getServerList = () => {
   if (state.useExternal == 'required') {
     return state.externalServerList[0]
@@ -50,8 +55,10 @@ export default () => ({
   getVersion: computed(() => state.version),
   getIconSet: computed(() => state.regionIconSet),
   getInfoUrl: computed(() => state.infoUrl),
+  isSafari: computed(() => state.safariBrowser),
   set,
   agree,
+  safariDetected,
   getServerList,
   toggleFullscreen
 })

src/pages/camera-mixin.js

@@ -1,3 +1,5 @@
+import { f7 } from 'framework7-vue'
+
 export default {
   methods: {
     async openCamera(imContain) {
@@ -38,6 +40,54 @@ export default {
       const tempCtx = tempCVS.getContext('2d')
       tempCtx.drawImage(vidViewer, 0, 0)
       this.getImage(tempCVS.toDataURL())
+    },
+    async videoFrameDetectWorker (vidData) {
+      const startDetection = () => {
+        createImageBitmap(vidData).then(imVideoFrame => {
+          this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
+        })
+      }
+      vidData.addEventListener('resize',startDetection,{once: true})
+      this.vidWorker.onmessage = (eVid) => {
+        if (eVid.data.error) {
+          console.log(eVid.data.message)
+          f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`)
+        } else if (this.videoAvailable) {
+          createImageBitmap(vidData).then(imVideoFrame => {
+            this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
+          })
+          if (eVid.data.coords) {
+            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+            for (var coord of eVid.data.coords) {
+              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / eVid.data.modelWidth) * imgWidth - 10
+              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / eVid.data.modelHeight) * imgHeight - 10
+              console.debug(`cx: ${pointX}, cy: ${pointY}`)
+              imageCtx.globalAlpha = coord[2]
+              imageCtx.drawImage(target, pointX, pointY, 20, 20)
+            }
+          }
+        }
+      }
+
+      const imCanvas = this.$refs.image_cvs
+      const imageCtx = imCanvas.getContext("2d")
+      const target = this.$refs.target_image
+      var imgWidth
+      var imgHeight
+      f7.utils.nextFrame(() => {
+        imCanvas.width = imCanvas.clientWidth
+        imCanvas.height = imCanvas.clientHeight
+        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+        const imgAspect = vidData.width / vidData.height
+        const rendAspect = imCanvas.width / imCanvas.height
+        if (imgAspect >= rendAspect) {
+          imgWidth = imCanvas.width
+          imgHeight = imCanvas.width / imgAspect
+        } else {
+          imgWidth = imCanvas.height * imgAspect
+          imgHeight = imCanvas.height
+        }
+      })
     }
   }
 }

src/pages/detect.vue

@@ -10,7 +10,7 @@
     </f7-navbar>
     <f7-block class="detect-grid">
       <div class="image-container" ref="image_container">
-        <SvgIcon v-if="!imageView.src && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)" @click="selectImage" />
+        <SvgIcon v-if="!imageView.src && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)"/>
         <div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
           <video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
           <f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
@@ -140,6 +140,8 @@
   import detectionMixin from './detection-mixin'
   import cameraMixin from './camera-mixin'
 
+  import detectionWorker from '@/assets/detect-worker.js?worker&inline'
+
   export default {
     mixins: [submitMixin, detectionMixin, cameraMixin],
     props: {
@@ -177,7 +179,9 @@
         videoDeviceAvailable: false,
         videoAvailable: false,
         cameraStream: null,
-        infoLinkPos: {}
+        infoLinkPos: {},
+        detectWorker: null,
+        vidWorker: null
       }
     },
     setup() {
@@ -186,7 +190,6 @@
     created () {
       let loadOtherSettings = localStorage.getItem('otherSettings')
       if (loadOtherSettings) this.otherSettings = JSON.parse(loadOtherSettings)
-      let modelRoot = this.isCordova ? 'https://localhost' : '.'
       this.detectorName = this.f7route.params.region
       switch (this.detectorName) {
         case 'thorax':
@@ -202,9 +205,9 @@
           this.activeRegion = 3
           break;
       }
-      this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
-      this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
-      fetch(`${this.isCordova ? 'https://localhost' : '.'}/models/${this.detectorName}/classes.json`)
+      this.modelLocation = URL.parse(`../models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`,import.meta.url).href
+      this.miniLocation = URL.parse(`../models/${this.detectorName}-mini/model.json`,import.meta.url).href
+      fetch(URL.parse(`../models/${this.detectorName}/classes.json`,import.meta.url).href)
         .then((mod) => { return mod.json() })
         .then((classes) => {
           this.classesList = classes
@@ -214,18 +217,41 @@
       if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
     },
     mounted () {
+      this.detectWorker = new detectionWorker()
+      this.detectWorker.onmessage = (eMount) => {
+        self = this
+        if (eMount.data.error) {
+          console.log(eMount.data.message)
+          f7.dialog.alert(`ALVINN AI model error: ${eMount.data.message}`)
+        }
+        self.modelLoading = false
+      }
+      this.vidWorker = new detectionWorker()
+      this.vidWorker.onmessage = (eMount) => {
+        self = this
+        if (eMount.data.error) {
+          console.log(eMount.data.message)
+          f7.dialog.alert(`ALVINN AI nano model error: ${eMount.data.message}`)
+        }
+      }
+
       if (this.serverSettings && this.serverSettings.use) {
         this.getRemoteLabels()
         this.modelLoading = false
       } else {
         this.modelLoading = true
-        this.loadModel(this.modelLocation, true).then(() => {
-          this.modelLoading = false
-        }).catch((e) => {
-          console.log(e.message)
-          f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
-          this.modelLoading = false
-        })
+        if (this.isSafari) {
+          this.loadModel(this.modelLocation, true).then(() => {
+            this.modelLoading = false
+          }).catch((e) => {
+            console.log(e.message)
+            f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
+            this.modelLoading = false
+          })
+        } else {
+          this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true})
+          this.vidWorker.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
+        }
       }
       window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') }
     },
@@ -278,7 +304,7 @@
       infoLinkTarget () {
         if (!this.getInfoUrl) return ''
         let structure = this.showResults.find( r => r.resultIndex == this.selectedChip)
-        return this.getInfoUrl + structure.label.replaceAll(' ','_')
+        return structure ? this.getInfoUrl + structure.label.replaceAll(' ','_') : ''
       }
     },
     methods: {
@@ -287,12 +313,50 @@
         return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
       },
       async setData () {
-        if (this.reloadModel) {
-          await this.loadModel(this.modelLocation)
-          this.reloadModel = false
+        this.detectWorker.onmessage = (eDetect) => {
+          self = this
+          if (eDetect.data.error) {
+            self.detecting = false
+            self.resultData = {}
+            loadFailure()
+            f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
+          } else if (eDetect.data.success == 'detection') {
+            self.detecting = false
+            self.resultData = eDetect.data.detections
+            if (self.resultData) {
+              self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name})
+            }
+            self.uploadDirty = true
+          } else if (eDetect.data.success == 'model') {
+            self.reloadModel = false
+            loadSuccess()
+          }
         }
+
+        let loadSuccess = null
+        let loadFailure = null
+        let modelReloading = null
+        if (this.isSafari && this.reloadModel) {
+          await this.loadModel(this.modelLocation)
+          this.reloadModel = false          
+        } else {
+          modelReloading = new Promise((res, rej) => {
+            loadSuccess = res
+            loadFailure = rej
+            if (this.reloadModel) {
+              this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
+            } else {
+              loadSuccess()
+            }
+          })
+        }
+
         if (this.serverSettings && this.serverSettings.use) {
           this.remoteDetect()
+        } else if (!this.isSafari) {
+          Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
+            this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
+          })
         } else {
           this.localDetect(this.imageView).then(dets => {
             this.detecting = false
@@ -319,9 +383,10 @@
           navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
           return
         }
-        if (mode == "camera") {
+        if (mode == "camera" && !this.otherSettings.disableVideo) {
           this.videoAvailable = await this.openCamera(this.$refs.image_container)
           if (this.videoAvailable) {
+            this.selectedChip = -1
             this.imageLoaded = false
             this.imageView.src = null
             this.$refs.image_cvs.style['background-image'] = 'none'
@@ -330,8 +395,10 @@
             var vidElement = this.$refs.vid_viewer
             vidElement.width = trackDetails.width
             vidElement.height = trackDetails.height
-            if (!this.otherSettings.disableVideo) {
+            if (this.isSafari) {
               this.videoFrameDetect(vidElement)
+            } else {
+              this.videoFrameDetectWorker(vidElement)
             }
             return
           }
@@ -445,13 +512,9 @@
         }).then( () => {
           const [imCanvas, _]  = this.resetView()
           imCanvas.style['background-image'] = `url(${this.imageView.src})`
-          /******
-           * setTimeout is not a good solution, but it's the only way
-           * I can find to not cut off drawing of the canvas background
-           ******/
-          setTimeout(() => {
+          f7.utils.nextFrame(() => {
             this.setData()
-          }, 1)
+          })
         }).catch((e) => {
           console.log(e.message)
           f7.dialog.alert(`Error loading image: ${e.message}`)

src/pages/detection-mixin.js

@@ -25,7 +25,7 @@ export default {
       return model
     },
     async localDetect(imageData) {
-      console.time('pre-process')
+      console.time('mx: pre-process')
       const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
       let gTense = null
       const input = tf.tidy(() => {
@@ -33,15 +33,15 @@ export default {
         return tf.concat([gTense,gTense,gTense],3)
       })
       tf.dispose(gTense)
-      console.timeEnd('pre-process')
+      console.timeEnd('mx: pre-process')
 
-      console.time('run prediction')
+      console.time('mx: run prediction')
       const res = model.predict(input)
       const tRes = tf.transpose(res,[0,2,1])
       const rawRes = tRes.arraySync()[0]
-      console.timeEnd('run prediction')
+      console.timeEnd('mx: run prediction')
 
-      console.time('post-process')
+      console.time('mx: post-process')
       const outputSize = res.shape[1]
       let rawBoxes = []
       let rawScores = []
@@ -105,7 +105,7 @@ export default {
       }
       tf.dispose(res)
       tf.dispose(input)
-      console.timeEnd('post-process')
+      console.timeEnd('mx: post-process')
 
       return output || { detections: [] }
     },
@@ -194,7 +194,7 @@ export default {
         imgHeight = imCanvas.height
       }
       while (this.videoAvailable) {
-        console.time('frame-process')
+        console.time('mx: frame-process')
         try {
           const input = tf.tidy(() => {
             return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
@@ -228,7 +228,7 @@ export default {
         } catch (e) {
           console.log(e)
         }
-        console.timeEnd('frame-process')
+        console.timeEnd('mx: frame-process')
         await tf.nextFrame();
       }
     }

src/pages/specs.vue

@@ -8,6 +8,7 @@
         <f7-block-title medium>Details</f7-block-title>
         <f7-list>
           <f7-list-item title="Version" :after="alvinnVersion"></f7-list-item>
+          <f7-list-item v-if="isSafari" title="Safari" after="Workers disabled"></f7-list-item>
         </f7-list>
         <f7-block-title medium>Models</f7-block-title>
         <f7-list style="width: 100%;">
@@ -52,6 +53,7 @@
         miniHeadneckDetails: {},
         alvinnVersion: store().getVersion,
         isCordova: !!window.cordova,
+        isSafari: store().isSafari,
         otherSettings: {}
       }
     },