Clean up shared worker addition

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

src/assets/detect-worker.js

@@ -10,22 +10,22 @@ self.onconnect = (e) => {
     switch (e.data.call) {
       case 'loadModel':
         loadModel('.' + e.data.weights,e.data.preload).then(() => {
-          port.postMessage({success: true})
+          port.postMessage({success: 'model'})
         }).catch((err) => {
           port.postMessage({error: true, message: err.message})
         })
         break
       case 'localDetect':
         localDetect(e.data.image).then((dets) => {
-          port.postMessage({success: true, detections: dets})
+          port.postMessage({success: 'detection', detections: dets})
         }).catch((err) => {
           port.postMessage({error: true, message: err.message})
         })
         e.data.image.close()
         break
       case 'videoFrame':
-        videoFrame(e.data.image).then((franeDet) =>{
-          port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH})
+        videoFrame(e.data.image).then((frameDet) =>{
+          port.postMessage({succes: 'frame', coords: frameDet.cds, modelWidth: frameDet.mW, modelHeight: frameDet.mH})
         }).catch((err) => {
           port.postMessage({error: true, message: err.message})
         })
@@ -149,72 +149,6 @@ async function localDetect(imageData) {
   return output || { detections: [] }
 }
 
-function getRemoteLabels() {
-  var self = this
-  var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
-  var xhr = new XMLHttpRequest()
-  xhr.open("GET", modelURL)
-  xhr.setRequestHeader('Content-Type', 'application/json')
-  xhr.timeout = 10000
-  xhr.ontimeout = this.remoteTimeout
-  xhr.onload = function () {
-    if (this.status !== 200) {
-      console.log(xhr.response)
-      const errorResponse = JSON.parse(xhr.response)
-      f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
-      return
-    }
-    var detectors = JSON.parse(xhr.response).detectors
-    var findLabel = detectors
-      .find( d => { return d.name == self.detectorName } )?.labels
-      .filter( l => { return l != "" } ).sort()
-      .map( l => { return {'name': l, 'detect': true} } )
-    self.detectorLabels = findLabel || []
-  }
-  xhr.onerror = function (e) {
-    f7.dialog.alert('ALVINN has encountered an unknown server error')
-    return
-  }
-  
-  xhr.send()
-}
-
-function remoteDetect() {
-  var self = this
-  var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
-  var xhr = new XMLHttpRequest()
-  xhr.open("POST", modelURL)
-  xhr.timeout = 10000
-  xhr.ontimeout = this.remoteTimeout
-  xhr.setRequestHeader('Content-Type', 'application/json')
-  xhr.onload = function () {
-    self.detecting = false
-    if (this.status !== 200) {
-      console.log(xhr.response)
-      const errorResponse = JSON.parse(xhr.response)
-      f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
-      return;
-    }
-    self.resultData = JSON.parse(xhr.response)
-    self.uploadDirty = true
-  }
-  
-  var doodsData = {
-    "detector_name": this.detectorName,
-    "detect": {
-      "*": 1
-    },
-    "data": this.imageView.src.split(',')[1]
-  }
-
-  xhr.send(JSON.stringify(doodsData))
-}
-
-function remoteTimeout () {
-  this.detecting = false
-  f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
-}
-
 async function videoFrame (vidData) {
   const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
   console.time('frame-process')

src/pages/camera-mixin.js

@@ -42,7 +42,7 @@ export default {
       this.getImage(tempCVS.toDataURL())
     },
     async videoFrameDetect (vidData) {
-      const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
+      const vidWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
       vidWorker.port.onmessage = (eVid) => {
         self = this
         if (eVid.data.error) {

src/pages/detect.vue

@@ -177,7 +177,8 @@
         videoDeviceAvailable: false,
         videoAvailable: false,
         cameraStream: null,
-        infoLinkPos: {}
+        infoLinkPos: {},
+        workerScript: null
       }
     },
     setup() {
@@ -204,7 +205,7 @@
       }
       this.modelLocation = `${modelRoot}/models/${this.detectorName}${this.otherSettings.mini ? '-mini' : ''}/model.json`
       this.miniLocation = `${modelRoot}/models/${this.detectorName}-mini/model.json`
-      fetch(`${this.isCordova ? 'https://localhost' : '.'}/models/${this.detectorName}/classes.json`)
+      fetch(`${modelRoot}/models/${this.detectorName}/classes.json`)
         .then((mod) => { return mod.json() })
         .then((classes) => {
           this.classesList = classes
@@ -214,7 +215,7 @@
       if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
     },
     mounted () {
-      const mountWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
+      const mountWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
       mountWorker.port.onmessage = (eMount) => {
         self = this
         if (eMount.data.error) {
@@ -291,32 +292,43 @@
         return `--chip-media-gradient: conic-gradient(from ${270 - (confFactor * 360 / 2)}deg, hsl(${confFactor * 120}deg, 100%, 50%) ${confFactor}turn, hsl(${confFactor * 120}deg, 50%, 66%) ${confFactor}turn)`
       },
       async setData () {
-        const detectWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
+        const detectWorker = new SharedWorker('../assets/detect-worker.js',{type: 'module'})
         detectWorker.port.onmessage = (eDetect) => {
           self = this
           if (eDetect.data.error) {
             self.detecting = false
             self.resultData = {}
             f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
-          } else {
+          } else if (eDetect.data.success == 'detection') {
             self.detecting = false
             self.resultData = eDetect.data.detections
             if (self.resultData) {
               self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name})
             }
             self.uploadDirty = true
+          } else if (eDetect.data.success == 'model') {
+            this.reloadModel = false
+            loadSuccess(true)
           }
         }
 
-        if (this.reloadModel) {
-          await this.loadModel(this.modelLocation)
-          this.reloadModel = false
-        }
+        let loadSuccess = null
+        let loadFailure = null
+        let modelReloading = new Promise((res, rej) => {
+          loadSuccess = res
+          loadFailure = rej
+          if (this.reloadModel) {
+            detectWorker.port.postMessage({call: 'loadModel', weights: this.modelLocation})
+          } else {
+            loadSuccess(true)
+          }
+        })
+
         if (this.serverSettings && this.serverSettings.use) {
           this.remoteDetect()
         } else {
-          createImageBitmap(this.imageView).then(imData => {
-            detectWorker.port.postMessage({call: 'localDetect', image: imData}, [imData])
+          Promise.all([modelReloading,createImageBitmap(this.imageView)]).then(res => {
+            detectWorker.port.postMessage({call: 'localDetect', image: res[1]}, [res[1]])
           })
         }
       },

src/pages/detection-mixin.js

@@ -1,114 +1,7 @@
-import * as tf  from '@tensorflow/tfjs'
 import { f7 } from 'framework7-vue'
 
-let model = null
-
 export default {
   methods: {
-    async loadModel(weights, preload) {
-      if (model && model.modelURL == weights) {
-        return model
-      } else if (model) {
-        tf.dispose(model)
-      }
-      model = await tf.loadGraphModel(weights)
-      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-      /*****************
-       * If preloading then run model
-       * once on fake data to preload
-       * weights for a faster response
-       *****************/
-      if (preload) {
-        const dummyT = tf.ones([1,modelWidth,modelHeight,3])
-        model.predict(dummyT)
-      }
-      return model
-    },
-    async localDetect(imageData) {
-      console.time('pre-process')
-      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-      let gTense = null
-      const input = tf.tidy(() => {
-        gTense = tf.image.rgbToGrayscale(tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight])).div(255.0).expandDims(0)
-        return tf.concat([gTense,gTense,gTense],3)
-      })
-      tf.dispose(gTense)
-      console.timeEnd('pre-process')
-
-      console.time('run prediction')
-      const res = model.predict(input)
-      const tRes = tf.transpose(res,[0,2,1])
-      const rawRes = tRes.arraySync()[0]
-      console.timeEnd('run prediction')
-
-      console.time('post-process')
-      const outputSize = res.shape[1]
-      let rawBoxes = []
-      let rawScores = []
-
-      for (var i = 0; i < rawRes.length; i++) {
-        var getScores = rawRes[i].slice(4)
-        if (getScores.every( s => s < .05)) { continue }
-        var getBox = rawRes[i].slice(0,4)
-        var boxCalc = [
-          (getBox[0] - (getBox[2] / 2)) / modelWidth,
-          (getBox[1] - (getBox[3] / 2)) / modelHeight,
-          (getBox[0] + (getBox[2] / 2)) / modelWidth,
-          (getBox[1] + (getBox[3] / 2)) / modelHeight,
-        ]
-        rawBoxes.push(boxCalc)
-        rawScores.push(getScores)
-      }
-
-      if (rawBoxes.length > 0) {
-        const tBoxes = tf.tensor2d(rawBoxes)
-        let tScores = null
-        let resBoxes = null
-        let validBoxes = []
-        let structureScores = null
-        let boxes_data = []
-        let scores_data = []
-        let classes_data = []
-        for (var c = 0; c < outputSize - 4; c++) {
-          structureScores = rawScores.map(x => x[c])
-          tScores = tf.tensor1d(structureScores)
-          resBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tScores,10,0.5,.05)
-          validBoxes = resBoxes.dataSync()
-          tf.dispose(resBoxes)
-          if (validBoxes) {
-            boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
-            var outputScores = structureScores.filter( (_, idx) => validBoxes.includes(idx))
-            scores_data.push(...outputScores)
-            classes_data.push(...outputScores.fill(c))
-          }
-        }
-
-        validBoxes = []
-        tf.dispose(tBoxes)
-        tf.dispose(tScores)
-        tf.dispose(tRes)
-        const valid_detections_data = classes_data.length
-        var output = {
-          detections: []
-        }
-        for (var i =0; i < valid_detections_data; i++) {
-          var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
-          output.detections.push({
-            "top": dTop,
-            "left": dLeft,
-            "bottom": dBottom,
-            "right": dRight,
-            "label": this.detectorLabels[classes_data[i]].name,
-            "confidence": scores_data[i] * 100
-          })
-        }
-      }
-      tf.dispose(res)
-      tf.dispose(input)
-      console.timeEnd('post-process')
-
-      return output || { detections: [] }
-    },
     getRemoteLabels() {
       var self = this
       var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`