Split detect.vue (#122)

Closes: #112

New mixins for camera and detection (remote and local) and new css for detection page.

Reviewed-on: #122

src/pages/detection-mixin.js

@@ -0,0 +1,147 @@
+import * as tf  from '@tensorflow/tfjs'
+import { f7 } from 'framework7-vue'
+
+var model = null
+
+export default {
+  methods: {
+    async loadModel(weights) {
+      model = await tf.loadGraphModel(weights).then(graphModel => {
+        return graphModel
+      })
+  },
+    async localDetect(imageData) {
+      console.time('pre-process')
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3);
+      const input = tf.tidy(() => {
+        return tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
+      })
+      console.timeEnd('pre-process')
+
+      console.time('run prediction')
+      const res = model.predict(input)
+      console.timeEnd('run prediction')
+
+      console.time('post-process')
+      const detectAttempts = res.shape[2]
+      const outputSize = res.shape[1]
+      const rawRes = tf.transpose(res,[0,2,1]).dataSync()
+      let rawBoxes = []
+      let rawScores = []
+
+      for (var i = 0; i < detectAttempts; i++) {
+        var getBox = rawRes.slice((i * outputSize),(i * outputSize) + 4)
+        var boxCalc = [
+          (getBox[0] - (getBox[2] / 2)) / modelWidth,
+          (getBox[1] - (getBox[3] / 2)) / modelHeight,
+          (getBox[0] + (getBox[2] / 2)) / modelWidth,
+          (getBox[1] + (getBox[3] / 2)) / modelHeight,
+        ]
+        rawBoxes.push(boxCalc)
+        rawScores.push(rawRes.slice((i * outputSize) + 4,(i + 1) * outputSize))
+      }
+      const tBoxes = tf.tensor2d(rawBoxes)
+      let tScores = null
+      let boxes_data = []
+      let scores_data = []
+      let classes_data = []
+      for (var c = 0; c < outputSize - 4; c++) {
+        tScores = rawScores.map(x => x[c])
+        var validBoxes = await tf.image.nonMaxSuppressionAsync(tBoxes,tf.tensor1d(tScores),10,0.5,.05)
+        validBoxes = validBoxes.dataSync()
+        if (validBoxes) {
+          boxes_data.push(...rawBoxes.filter( (_, idx) => validBoxes.includes(idx)))
+          var outputScores = tScores.filter( (_, idx) => validBoxes.includes(idx))
+          scores_data.push(...outputScores)
+          classes_data.push(...outputScores.fill(c))
+        }
+      }
+
+      const valid_detections_data = classes_data.length
+      var output = {
+        detections: []
+      }
+      for (var i =0; i < valid_detections_data; i++) {
+        var [dLeft, dTop, dRight, dBottom] = boxes_data[i]
+        output.detections.push({
+          "top": dTop,
+          "left": dLeft,
+          "bottom": dBottom,
+          "right": dRight,
+          "label": this.detectorLabels[classes_data[i]].name,
+          "confidence": scores_data[i] * 100
+        })
+      }
+
+      tf.dispose(res)
+      tf.dispose(tBoxes)
+      console.timeEnd('post-process')
+
+      return output
+    },
+    getRemoteLabels() {
+      var self = this
+      var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
+      var xhr = new XMLHttpRequest()
+      xhr.open("GET", modelURL)
+      xhr.setRequestHeader('Content-Type', 'application/json')
+      xhr.timeout = 10000
+      xhr.ontimeout = this.remoteTimeout
+      xhr.onload = function () {
+        if (this.status !== 200) {
+          console.log(xhr.response)
+          const errorResponse = JSON.parse(xhr.response)
+          f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
+          return
+        }
+        var detectors = JSON.parse(xhr.response).detectors
+        var findLabel = detectors
+          .find( d => { return d.name == self.detectorName } )?.labels
+          .filter( l => { return l != "" } ).sort()
+          .map( l => { return {'name': l, 'detect': true} } )
+        self.detectorLabels = findLabel || []
+      }
+      xhr.onerror = function (e) {
+        f7.dialog.alert('ALVINN has encountered an unknown server error')
+        return
+      }
+      
+      xhr.send()
+    },
+    remoteDetect() {
+      var self = this
+      var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
+      var xhr = new XMLHttpRequest()
+      xhr.open("POST", modelURL)
+      xhr.timeout = 10000
+      xhr.ontimeout = this.remoteTimeout
+      xhr.setRequestHeader('Content-Type', 'application/json')
+      xhr.onload = function () {
+        self.detecting = false
+        if (this.status !== 200) {
+          console.log(xhr.response)
+          const errorResponse = JSON.parse(xhr.response)
+          f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
+          return;
+        }
+        self.resultData = JSON.parse(xhr.response)
+        self.uploadDirty = true
+      }
+      
+      var doodsData = {
+        "detector_name": this.detectorName,
+        "detect": {
+          "*": 1
+        },
+        "data": this.imageView.src.split(',')[1]
+      }
+
+      xhr.send(JSON.stringify(doodsData))
+    },
+    remoteTimeout () {
+      this.detecting = false
+      f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
+    }
+
+  }
+}