Cleanup for 0.5.0 alpha release

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

cordova/config.xml

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<widget id="edu.midwestern.alvinn" version="0.5.0-rc" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
+<widget id="edu.midwestern.alvinn" version="0.5.0-alpha" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
     <name>ALVINN</name>
     <description>Anatomy Lab Visual Identification Neural Network.</description>
     <author email="jgeorg@midwestern.edu" href="https://midwestern.edu">

cordova/package.json

@@ -1,7 +1,7 @@
 {
   "name": "edu.midwestern.alvinn",
   "displayName": "ALVINN",
-  "version": "0.5.0-rc",
+  "version": "0.5.0-alpha",
   "description": "Anatomy Lab Visual Identification Neural Network.",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "alvinn",
   "private": true,
-  "version": "0.5.0-rc",
+  "version": "0.5.0-alpha",
   "description": "ALVINN",
   "repository": "",
   "license": "UNLICENSED",
@@ -14,7 +14,8 @@
     "cordova-ios": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run ios",
     "build-cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova build android",
     "cordova-android": "cross-env TARGET=cordova cross-env NODE_ENV=production vite build && node ./build/build-cordova.js && cd cordova && cordova run android",
-    "postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/"
+    "postinstall": "cpy --flat ./node_modules/framework7-icons/fonts/*.* ./src/fonts/",
+    "preview": "vite preview"
   },
   "browserslist": [
     "IOS >= 15",

src/assets/detect-worker.js

@@ -5,7 +5,7 @@ let model = null
 onmessage = function (e) {
   switch (e.data.call) {
     case 'loadModel':
-      loadModel('.' + e.data.weights,e.data.preload).then(() => {
+      loadModel(e.data.weights,e.data.preload).then(() => {
         postMessage({success: 'model'})
       }).catch((err) => {
         postMessage({error: true, message: err.message})
@@ -57,7 +57,7 @@ async function loadModel(weights, preload) {
 }
 
 async function localDetect(imageData) {
-  console.time('pre-process')
+  console.time('sw: pre-process')
   const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
   let gTense = null
   const input = tf.tidy(() => {
@@ -65,15 +65,15 @@ async function localDetect(imageData) {
     return tf.concat([gTense,gTense,gTense],3)
   })
   tf.dispose(gTense)
-  console.timeEnd('pre-process')
+  console.timeEnd('sw: pre-process')
 
-  console.time('run prediction')
+  console.time('sw: run prediction')
   const res = model.predict(input)
   const tRes = tf.transpose(res,[0,2,1])
   const rawRes = tRes.arraySync()[0]
-  console.timeEnd('run prediction')
+  console.timeEnd('sw: run prediction')
 
-  console.time('post-process')
+  console.time('sw: post-process')
   const outputSize = res.shape[1]
   let rawBoxes = []
   let rawScores = []
@@ -138,14 +138,14 @@ async function localDetect(imageData) {
   }
   tf.dispose(res)
   tf.dispose(input)
-  console.timeEnd('post-process')
+  console.timeEnd('sw: post-process')
 
   return output || { detections: [] }
 }
 
 async function videoFrame (vidData) {
   const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-  console.time('frame-process')
+  console.time('sw: frame-process')
   let rawCoords = []
   try {
     const input = tf.tidy(() => {
@@ -171,6 +171,6 @@ async function videoFrame (vidData) {
   } catch (e) {
     console.log(e)
   }
-  console.timeEnd('frame-process')
+  console.timeEnd('sw: frame-process')
   return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
 }

src/components/app.vue

@@ -79,6 +79,7 @@
           .then((mod) => { return mod.text() })
         this.siteConf = YAML.parse(confText)
       }
+      if (window.safari !== undefined) {store().safariDetected()}
       const loadSiteSettings = localStorage.getItem('siteSettings')
       if (loadSiteSettings) {
         let loadedSettings = JSON.parse(loadSiteSettings)

src/js/store.js

@@ -4,12 +4,13 @@ const state = reactive({
     disclaimerAgreement: false,
     enabledRegions: ['thorax','abdomen','limbs','head'],
     regionIconSet: Math.floor(Math.random() * 3) + 1,
-    version: '0.5.0-rc',
+    version: '0.5.0-alpha',
     fullscreen: false,
     useExternal: 'optional',
     siteDemo: false,
     externalServerList: [],
-    infoUrl: false
+    infoUrl: false,
+    safariBrowser: false
   })
 
 const set = (config, confObj) => {
@@ -21,6 +22,10 @@ const agree = () => {
   state.disclaimerAgreement = true
 }
 
+const safariDetected = () => {
+  state.safariBrowser = true
+}
+
 const getServerList = () => {
   if (state.useExternal == 'required') {
     return state.externalServerList[0]
@@ -50,8 +55,10 @@ export default () => ({
   getVersion: computed(() => state.version),
   getIconSet: computed(() => state.regionIconSet),
   getInfoUrl: computed(() => state.infoUrl),
+  isSafari: computed(() => state.safariBrowser),
   set,
   agree,
+  safariDetected,
   getServerList,
   toggleFullscreen
 })

src/pages/camera-mixin.js

@@ -41,7 +41,7 @@ export default {
       tempCtx.drawImage(vidViewer, 0, 0)
       this.getImage(tempCVS.toDataURL())
     },
-    async videoFrameDetect (vidData) {
+    async videoFrameDetectWorker (vidData) {
       const startDetection = () => {
         createImageBitmap(vidData).then(imVideoFrame => {
           this.vidWorker.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])

src/pages/detect.vue

@@ -140,7 +140,7 @@
   import detectionMixin from './detection-mixin'
   import cameraMixin from './camera-mixin'
 
-  import detectionWorker from '../assets/detect-worker.js?worker&inline'
+  import detectionWorker from '@/assets/detect-worker.js?worker&inline'
 
   export default {
     mixins: [submitMixin, detectionMixin, cameraMixin],
@@ -190,7 +190,6 @@
     created () {
       let loadOtherSettings = localStorage.getItem('otherSettings')
       if (loadOtherSettings) this.otherSettings = JSON.parse(loadOtherSettings)
-      let modelRoot = this.isCordova ? 'https://localhost' : '.'
       this.detectorName = this.f7route.params.region
       switch (this.detectorName) {
         case 'thorax':
@@ -206,9 +205,9 @@
           this.activeRegion = 3
           break;
       }
-      this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
-      this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
-      fetch(`${modelRoot}/models/${this.detectorName}/classes.json`)
+      this.modelLocation = URL.parse(`../models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`,import.meta.url).href
+      this.miniLocation = URL.parse(`../models/${this.detectorName}-mini/model.json`,import.meta.url).href
+      fetch(URL.parse(`../models/${this.detectorName}/classes.json`,import.meta.url).href)
         .then((mod) => { return mod.json() })
         .then((classes) => {
           this.classesList = classes
@@ -241,8 +240,18 @@
         this.modelLoading = false
       } else {
         this.modelLoading = true
-        this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true})
-        this.vidWorker.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
+        if (this.isSafari) {
+          this.loadModel(this.modelLocation, true).then(() => {
+            this.modelLoading = false
+          }).catch((e) => {
+            console.log(e.message)
+            f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
+            this.modelLoading = false
+          })
+        } else {
+          this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation, preload: true})
+          this.vidWorker.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
+        }
       }
       window.onresize = (e) => { if (this.$refs.image_cvs) this.selectChip('redraw') }
     },
@@ -304,7 +313,6 @@
         return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
       },
       async setData () {
-        //const detectWorker = new detectionWorker()
         this.detectWorker.onmessage = (eDetect) => {
           self = this
           if (eDetect.data.error) {
@@ -327,22 +335,39 @@
 
         let loadSuccess = null
         let loadFailure = null
-        let modelReloading = new Promise((res, rej) => {
-          loadSuccess = res
-          loadFailure = rej
-          if (this.reloadModel) {
-            this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
-          } else {
-            loadSuccess()
-          }
-        })
+        let modelReloading = null
+        if (this.isSafari && this.reloadModel) {
+          await this.loadModel(this.modelLocation)
+          this.reloadModel = false          
+        } else {
+          modelReloading = new Promise((res, rej) => {
+            loadSuccess = res
+            loadFailure = rej
+            if (this.reloadModel) {
+              this.detectWorker.postMessage({call: 'loadModel', weights: this.modelLocation})
+            } else {
+              loadSuccess()
+            }
+          })
+        }
 
         if (this.serverSettings && this.serverSettings.use) {
           this.remoteDetect()
-        } else {
+        } else if (!this.isSafari) {
           Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
             this.detectWorker.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
           })
+        } else {
+          this.localDetect(this.imageView).then(dets => {
+            this.detecting = false
+            this.resultData = dets
+            this.uploadDirty = true
+          }).catch((e) => {
+            console.log(e.message)
+            this.detecting = false
+            this.resultData = {}
+            f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
+          })
         }
       },
       selectAll (ev) {
@@ -358,7 +383,7 @@
           navigator.camera.getPicture(this.getImage, this.onFail, { quality: 50, destinationType: Camera.DestinationType.DATA_URL, correctOrientation: true });
           return
         }
-        if (mode == "camera") {
+        if (mode == "camera" && !this.otherSettings.disableVideo) {
           this.videoAvailable = await this.openCamera(this.$refs.image_container)
           if (this.videoAvailable) {
             this.selectedChip = -1
@@ -370,8 +395,10 @@
             var vidElement = this.$refs.vid_viewer
             vidElement.width = trackDetails.width
             vidElement.height = trackDetails.height
-            if (!this.otherSettings.disableVideo) {
+            if (this.isSafari) {
               this.videoFrameDetect(vidElement)
+            } else {
+              this.videoFrameDetectWorker(vidElement)
             }
             return
           }
@@ -485,13 +512,9 @@
         }).then( () => {
           const [imCanvas, _]  = this.resetView()
           imCanvas.style['background-image'] = `url(${this.imageView.src})`
-          /******
-           * setTimeout is not a good solution, but it's the only way
-           * I can find to not cut off drawing of the canvas background
-           ******/
-//          setTimeout(() => {
+          f7.utils.nextFrame(() => {
             this.setData()
-//          }, 1)
+          })
         }).catch((e) => {
           console.log(e.message)
           f7.dialog.alert(`Error loading image: ${e.message}`)

src/pages/detection-mixin.js

@@ -1,7 +1,114 @@
+import * as tf  from '@tensorflow/tfjs'
 import { f7 } from 'framework7-vue'
 
+let model = null
+
 export default {
   methods: {
+    async loadModel(weights, preload) {
+      if (model && model.modelURL == weights) {
+        return model
+      } else if (model) {
+        tf.dispose(model)
+      }
+      model = await tf.loadGraphModel(weights)
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+      /*****************
+       * If preloading then run model
+       * once on fake data to preload
+       * weights for a faster response
+       *****************/
+      if (preload) {
+        const dummyT = tf.ones([1,modelWidth,modelHeight,3])
+        model.predict(dummyT)
+      }
+      return model
+    },
+    async localDetect(imageData) {
+      console.time('mx: pre-process')
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+      let gTense = null
+      const input = tf.tidy(() => {
+        gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
+        return tf.concat([gTense,gTense,gTense],3)
+      })
+      tf.dispose(gTense)
+      console.timeEnd('mx: pre-process')
+
+      console.time('mx: run prediction')
+      const res = model.predict(input)
+      const tRes = tf.transpose(res,[0,2,1])
+      const rawRes = tRes.arraySync()[0]
+      console.timeEnd('mx: run prediction')
+
+      console.time('mx: post-process')
+      const outputSize = res.shape[1]
+      let rawBoxes = []
+      let rawScores = []
+
+      for (var i = 0; i < rawRes.length; i++) {
+        var getScores = rawRes[i].slice(4)
+        if (getScores.every( s => s < .05)) { continue }
+        var getBox = rawRes[i].slice(0,4)
+        var boxCalc = [
+          (getBox[0] - (getBox[2] / 2)) / modelWidth,
+          (getBox[1] - (getBox[3] / 2)) / modelHeight,
+          (getBox[0] + (getBox[2] / 2)) / modelWidth,
+          (getBox[1] + (getBox[3] / 2)) / modelHeight,
+        ]
+        rawBoxes.push(boxCalc)
+        rawScores.push(getScores)
+      }
+
+      if (rawBoxes.length > 0) {
+        const tBoxes = tf.tensor2d(rawBoxes)
+        let tScores = null
+        let resBoxes = null
+        let validBoxes = []
+        let structureScores = null
+        let boxes_data = []
+        let scores_data = []
+        let classes_data = []
+        for (var c = 0; c < outputSize - 4; c++) {
+          structureScores = rawScores.map(x => x[c])
+          tScores = tf.tensor1d(structureScores)
+          resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
+          validBoxes = resBoxes.dataSync()
+          tf.dispose(resBoxes)
+          if (validBoxes) {
+            boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
+            var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
+            scores_data.push(...outputScores)
+            classes_data.push(...outputScores.fill(c))
+          }
+        }
+
+        validBoxes = []
+        tf.dispose(tBoxes)
+        tf.dispose(tScores)
+        tf.dispose(tRes)
+        const valid_detections_data = classes_data.length
+        var output = {
+          detections: []
+        }
+        for (var i =0; i < valid_detections_data; i++) {
+          var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
+          output.detections.push({
+            "top": dTop,
+            "left": dLeft,
+            "bottom": dBottom,
+            "right": dRight,
+            "label": this.detectorLabels[classes_data[i]].name,
+            "confidence": scores_data[i] * 100
+          })
+        }
+      }
+      tf.dispose(res)
+      tf.dispose(input)
+      console.timeEnd('mx: post-process')
+
+      return output || { detections: [] }
+    },
     getRemoteLabels() {
       var self = this
       var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
@@ -65,5 +172,65 @@ export default {
       this.detecting = false
       f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
     },
+    async videoFrameDetect (vidData) {
+      await this.loadModel(this.miniLocation)
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+      const imCanvas = this.$refs.image_cvs
+      const imageCtx = imCanvas.getContext("2d")
+      const target = this.$refs.target_image
+      await tf.nextFrame();
+      imCanvas.width = imCanvas.clientWidth
+      imCanvas.height = imCanvas.clientHeight
+      imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+      var imgWidth
+      var imgHeight
+      const imgAspect = vidData.width / vidData.height
+      const rendAspect = imCanvas.width / imCanvas.height
+      if (imgAspect >= rendAspect) {
+        imgWidth = imCanvas.width
+        imgHeight = imCanvas.width / imgAspect
+      } else {
+        imgWidth = imCanvas.height * imgAspect
+        imgHeight = imCanvas.height
+      }
+      while (this.videoAvailable) {
+        console.time('mx: frame-process')
+        try {
+          const input = tf.tidy(() => {
+            return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
+          })
+          const res = model.predict(input)
+          const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
+
+          let rawCoords = []
+          if (rawRes) {
+            for (var i = 0; i < rawRes.length; i++) {
+              let getScores = rawRes[i].slice(4) 
+              if (getScores.some( s => s > .5)) {
+                let foundTarget = rawRes[i].slice(0,2)
+                foundTarget.push(Math.max(...getScores))
+                rawCoords.push(foundTarget)
+              }
+            }
+
+            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+            for (var coord of rawCoords) {
+              console.log(`x: ${coord[0]}, y: ${coord[1]}`)
+              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
+              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
+              imageCtx.globalAlpha = coord[2]
+              imageCtx.drawImage(target, pointX, pointY, 20, 20)
+            }
+          }
+          tf.dispose(input)
+          tf.dispose(res)
+          tf.dispose(rawRes)
+        } catch (e) {
+          console.log(e)
+        }
+        console.timeEnd('mx: frame-process')
+        await tf.nextFrame();
+      }
+    }
   }
 }

src/pages/specs.vue

@@ -8,6 +8,7 @@
         <f7-block-title medium>Details</f7-block-title>
         <f7-list>
           <f7-list-item title="Version" :after="alvinnVersion"></f7-list-item>
+          <f7-list-item v-if="isSafari" title="Safari" after="Workers disabled"></f7-list-item>
         </f7-list>
         <f7-block-title medium>Models</f7-block-title>
         <f7-list style="width: 100%;">
@@ -52,6 +53,7 @@
         miniHeadneckDetails: {},
         alvinnVersion: store().getVersion,
         isCordova: !!window.cordova,
+        isSafari: store().isSafari,
         otherSettings: {}
       }
     },