Add worker video detection

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

src/js/detect-worker.js

@@ -23,6 +23,14 @@ self.onconnect = (e) => {
         })
         e.data.image.close()
         break
+      case 'videoFrame':
+        videoFrame(e.data.image).then((franeDet) =>{
+          port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH})
+        }).catch((err) => {
+          port.postMessage({error: true, message: err.message})
+        })
+        e.data.image.close()
+        break
       default:
         console.log('Worker message incoming:')
         console.log(e)
@@ -207,29 +215,10 @@ function remoteTimeout () {
   f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
 }
 
-async function videoFrameDetect (vidData) {
-  await this.loadModel(this.miniLocation)
+async function videoFrame (vidData) {
   const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-  const imCanvas = this.$refs.image_cvs
-  const imageCtx = imCanvas.getContext("2d")
-  const target = this.$refs.target_image
-  await tf.nextFrame();
-  imCanvas.width = imCanvas.clientWidth
-  imCanvas.height = imCanvas.clientHeight
-  imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-  var imgWidth
-  var imgHeight
-  const imgAspect = vidData.width / vidData.height
-  const rendAspect = imCanvas.width / imCanvas.height
-  if (imgAspect >= rendAspect) {
-    imgWidth = imCanvas.width
-    imgHeight = imCanvas.width / imgAspect
-  } else {
-    imgWidth = imCanvas.height * imgAspect
-    imgHeight = imCanvas.height
-  }
-  while (this.videoAvailable) {
   console.time('frame-process')
+  let rawCoords = []
   try {
     const input = tf.tidy(() => {
       return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
@@ -237,7 +226,6 @@ async function videoFrameDetect (vidData) {
     const res = model.predict(input)
     const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
 
-      let rawCoords = []
     if (rawRes) {
       for (var i = 0; i < rawRes.length; i++) {
         let getScores = rawRes[i].slice(4) 
@@ -248,14 +236,6 @@ async function videoFrameDetect (vidData) {
         }
       }
 
-        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-        for (var coord of rawCoords) {
-          console.log(`x: ${coord[0]}, y: ${coord[1]}`)
-          let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
-          let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
-          imageCtx.globalAlpha = coord[2]
-          imageCtx.drawImage(target, pointX, pointY, 20, 20)
-        }
     }
     tf.dispose(input)
     tf.dispose(res)
@@ -264,6 +244,5 @@ async function videoFrameDetect (vidData) {
     console.log(e)
   }
   console.timeEnd('frame-process')
-    await tf.nextFrame();
-  }
+  return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
 }

src/pages/camera-mixin.js

@@ -1,3 +1,5 @@
+import { f7 } from 'framework7-vue'
+
 export default {
   methods: {
     async openCamera(imContain) {
@@ -38,6 +40,51 @@ export default {
       const tempCtx = tempCVS.getContext('2d')
       tempCtx.drawImage(vidViewer, 0, 0)
       this.getImage(tempCVS.toDataURL())
+    },
+    async videoFrameDetect (vidData) {
+      const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
+      vidWorker.port.onmessage = (eVid) => {
+        self = this
+        if (eVid.data.error) {
+          console.log(eVid.data.message)
+          f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`)
+        } else if (this.videoAvailable) {
+          createImageBitmap(vidData).then(imVideoFrame => {
+            vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
+          })
+          if (eVid.data.coords) {
+            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+            for (var coord of eVid.data.coords) {
+              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / eVid.data.modelWidth) * imgWidth - 10
+              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / eVid.data.modelHeight) * imgHeight - 10
+              console.debug(`cx: ${pointX}, cy: ${pointY}`)
+              imageCtx.globalAlpha = coord[2]
+              imageCtx.drawImage(target, pointX, pointY, 20, 20)
+            }
+          }
+        }
+      }
+
+      vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
+      const imCanvas = this.$refs.image_cvs
+      const imageCtx = imCanvas.getContext("2d")
+      const target = this.$refs.target_image
+      var imgWidth
+      var imgHeight
+      f7.utils.nextFrame(() => {
+        imCanvas.width = imCanvas.clientWidth
+        imCanvas.height = imCanvas.clientHeight
+        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+        const imgAspect = vidData.width / vidData.height
+        const rendAspect = imCanvas.width / imCanvas.height
+        if (imgAspect >= rendAspect) {
+          imgWidth = imCanvas.width
+          imgHeight = imCanvas.width / imgAspect
+        } else {
+          imgWidth = imCanvas.height * imgAspect
+          imgHeight = imCanvas.height
+        }
+      })
     }
   }
 }

src/pages/detection-mixin.js

@@ -172,65 +172,5 @@ export default {
       this.detecting = false
       f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
     },
-    async videoFrameDetect (vidData) {
-      await this.loadModel(this.miniLocation)
-      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-      const imCanvas = this.$refs.image_cvs
-      const imageCtx = imCanvas.getContext("2d")
-      const target = this.$refs.target_image
-      await tf.nextFrame();
-      imCanvas.width = imCanvas.clientWidth
-      imCanvas.height = imCanvas.clientHeight
-      imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-      var imgWidth
-      var imgHeight
-      const imgAspect = vidData.width / vidData.height
-      const rendAspect = imCanvas.width / imCanvas.height
-      if (imgAspect >= rendAspect) {
-        imgWidth = imCanvas.width
-        imgHeight = imCanvas.width / imgAspect
-      } else {
-        imgWidth = imCanvas.height * imgAspect
-        imgHeight = imCanvas.height
-      }
-      while (this.videoAvailable) {
-        console.time('frame-process')
-        try {
-          const input = tf.tidy(() => {
-            return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
-          })
-          const res = model.predict(input)
-          const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
-
-          let rawCoords = []
-          if (rawRes) {
-            for (var i = 0; i < rawRes.length; i++) {
-              let getScores = rawRes[i].slice(4) 
-              if (getScores.some( s => s > .5)) {
-                let foundTarget = rawRes[i].slice(0,2)
-                foundTarget.push(Math.max(...getScores))
-                rawCoords.push(foundTarget)
-              }
-            }
-
-            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-            for (var coord of rawCoords) {
-              console.log(`x: ${coord[0]}, y: ${coord[1]}`)
-              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
-              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
-              imageCtx.globalAlpha = coord[2]
-              imageCtx.drawImage(target, pointX, pointY, 20, 20)
-            }
-          }
-          tf.dispose(input)
-          tf.dispose(res)
-          tf.dispose(rawRes)
-        } catch (e) {
-          console.log(e)
-        }
-        console.timeEnd('frame-process')
-        await tf.nextFrame();
-      }
-    }
   }
 }