Add real-time detection to camera stream (#143)

Closes: #30

When the camera is being used to find an image to capture, the region mini model now runs in real time to give an estimate of where there are identifiable structures.

Reviewed-on: #143

src/pages/detection-mixin.js

@@ -5,11 +5,23 @@ var model = null
 
 export default {
   methods: {
-    async loadModel(weights) {
+    async loadModel(weights, preload) {
+      if (model && model.modelURL == weights) {
+        return model
+      } else if (model) {
+        model.dispose()
+      }
       model = await tf.loadGraphModel(weights)
       const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-      const dummyT = tf.ones([1,modelWidth,modelHeight,3])
-      model.predict(dummyT) //Run model once to preload weights for better response time
+      /*****************
+       * If preloading then run model
+       * once on fake data to preload
+       * weights for a faster response
+       *****************/
+      if (preload) {
+        const dummyT = tf.ones([1,modelWidth,modelHeight,3])
+        model.predict(dummyT)
+      }
       return model
     },
     async localDetect(imageData) {
@@ -150,7 +162,60 @@ export default {
     remoteTimeout () {
       this.detecting = false
       f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
-    }
+    },
+    async videoFrameDetect (vidData) {
+      await this.loadModel(this.miniLocation)
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+      const imCanvas = this.$refs.image_cvs
+      const imageCtx = imCanvas.getContext("2d")
+      const target = this.$refs.target_image
+      await tf.nextFrame();
+      imCanvas.width = imCanvas.clientWidth
+      imCanvas.height = imCanvas.clientHeight
+      imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+      var imgWidth
+      var imgHeight
+      const imgAspect = vidData.clientWidth / vidData.clientHeight
+      const rendAspect = imCanvas.width / imCanvas.height
+      if (imgAspect >= rendAspect) {
+        imgWidth = imCanvas.width
+        imgHeight = imCanvas.width / imgAspect
+      } else {
+        imgWidth = imCanvas.height * imgAspect
+        imgHeight = imCanvas.height
+      }
+      while (this.videoAvailable) {
+        console.time('frame-process')
+        try {
+          const input = tf.tidy(() => {
+            return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
+          })
+          const res = model.predict(input)
+          const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
 
+          let rawCoords = []
+          if (rawRes) {
+            for (var i = 0; i < rawRes.length; i++) {
+              var getScores = rawRes[i].slice(4)
+              if (getScores.some( s => s > .5)) {
+                rawCoords.push(rawRes[i].slice(0,2))
+              }
+            }
+
+            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+            for (var coord of rawCoords) {
+              console.log(`x: ${coord[0]}, y: ${coord[1]}`)
+              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
+              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
+              imageCtx.drawImage(target, pointX, pointY, 20, 20)
+            }
+          }
+        } catch (e) {
+          console.log(e)
+        }
+        console.timeEnd('frame-process')
+        await tf.nextFrame();
+      }
+    }
   }
 }