Add worker video detection

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

src/js/detect-worker.js

@@ -23,6 +23,14 @@ self.onconnect = (e) => {
         })
         e.data.image.close()
         break
+      case 'videoFrame':
+        videoFrame(e.data.image).then((franeDet) =>{
+          port.postMessage({succes: true, coords: franeDet.cds, modelWidth: franeDet.mW, modelHeight: franeDet.mH})
+        }).catch((err) => {
+          port.postMessage({error: true, message: err.message})
+        })
+        e.data.image.close()
+        break
       default:
         console.log('Worker message incoming:')
         console.log(e)
@@ -207,63 +215,34 @@ function remoteTimeout () {
   f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
 }
 
-async function videoFrameDetect (vidData) {
+async function videoFrame (vidData) {
-  await this.loadModel(this.miniLocation)
   const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-  const imCanvas = this.$refs.image_cvs
+  console.time('frame-process')
-  const imageCtx = imCanvas.getContext("2d")
+  let rawCoords = []
-  const target = this.$refs.target_image
+  try {
-  await tf.nextFrame();
+    const input = tf.tidy(() => {
-  imCanvas.width = imCanvas.clientWidth
+      return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
-  imCanvas.height = imCanvas.clientHeight
+    })
-  imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+    const res = model.predict(input)
-  var imgWidth
+    const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
-  var imgHeight
-  const imgAspect = vidData.width / vidData.height
-  const rendAspect = imCanvas.width / imCanvas.height
-  if (imgAspect >= rendAspect) {
-    imgWidth = imCanvas.width
-    imgHeight = imCanvas.width / imgAspect
-  } else {
-    imgWidth = imCanvas.height * imgAspect
-    imgHeight = imCanvas.height
-  }
-  while (this.videoAvailable) {
-    console.time('frame-process')
-    try {
-      const input = tf.tidy(() => {
-        return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
-      })
-      const res = model.predict(input)
-      const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
 
-      let rawCoords = []
+    if (rawRes) {
-      if (rawRes) {
+      for (var i = 0; i < rawRes.length; i++) {
-        for (var i = 0; i < rawRes.length; i++) {
+        let getScores = rawRes[i].slice(4) 
-          let getScores = rawRes[i].slice(4) 
+        if (getScores.some( s => s > .5)) {
-          if (getScores.some( s => s > .5)) {
+          let foundTarget = rawRes[i].slice(0,2)
-            let foundTarget = rawRes[i].slice(0,2)
+          foundTarget.push(Math.max(...getScores))
-            foundTarget.push(Math.max(...getScores))
+          rawCoords.push(foundTarget)
-            rawCoords.push(foundTarget)
-          }
-        }
-
-        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-        for (var coord of rawCoords) {
-          console.log(`x: ${coord[0]}, y: ${coord[1]}`)
-          let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
-          let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
-          imageCtx.globalAlpha = coord[2]
-          imageCtx.drawImage(target, pointX, pointY, 20, 20)
         }
       }
-      tf.dispose(input)
+
-      tf.dispose(res)
-      tf.dispose(rawRes)
-    } catch (e) {
-      console.log(e)
     }
-    console.timeEnd('frame-process')
+    tf.dispose(input)
-    await tf.nextFrame();
+    tf.dispose(res)
+    tf.dispose(rawRes)
+  } catch (e) {
+    console.log(e)
   }
+  console.timeEnd('frame-process')
+  return {cds: rawCoords, mW: modelWidth, mH: modelHeight}
 }

src/pages/camera-mixin.js

@@ -1,3 +1,5 @@
+import { f7 } from 'framework7-vue'
+
 export default {
   methods: {
     async openCamera(imContain) {
@@ -38,6 +40,51 @@ export default {
       const tempCtx = tempCVS.getContext('2d')
       tempCtx.drawImage(vidViewer, 0, 0)
       this.getImage(tempCVS.toDataURL())
+    },
+    async videoFrameDetect (vidData) {
+      const vidWorker = new SharedWorker('../js/detect-worker.js',{type: 'module'})
+      vidWorker.port.onmessage = (eVid) => {
+        self = this
+        if (eVid.data.error) {
+          console.log(eVid.data.message)
+          f7.dialog.alert(`ALVINN AI model error: ${eVid.data.message}`)
+        } else if (this.videoAvailable) {
+          createImageBitmap(vidData).then(imVideoFrame => {
+            vidWorker.port.postMessage({call: 'videoFrame', image: imVideoFrame}, [imVideoFrame])
+          })
+          if (eVid.data.coords) {
+            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+            for (var coord of eVid.data.coords) {
+              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / eVid.data.modelWidth) * imgWidth - 10
+              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / eVid.data.modelHeight) * imgHeight - 10
+              console.debug(`cx: ${pointX}, cy: ${pointY}`)
+              imageCtx.globalAlpha = coord[2]
+              imageCtx.drawImage(target, pointX, pointY, 20, 20)
+            }
+          }
+        }
+      }
+
+      vidWorker.port.postMessage({call: 'loadModel', weights: this.miniLocation, preload: true})
+      const imCanvas = this.$refs.image_cvs
+      const imageCtx = imCanvas.getContext("2d")
+      const target = this.$refs.target_image
+      var imgWidth
+      var imgHeight
+      f7.utils.nextFrame(() => {
+        imCanvas.width = imCanvas.clientWidth
+        imCanvas.height = imCanvas.clientHeight
+        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+        const imgAspect = vidData.width / vidData.height
+        const rendAspect = imCanvas.width / imCanvas.height
+        if (imgAspect >= rendAspect) {
+          imgWidth = imCanvas.width
+          imgHeight = imCanvas.width / imgAspect
+        } else {
+          imgWidth = imCanvas.height * imgAspect
+          imgHeight = imCanvas.height
+        }
+      })
     }
   }
 }

src/pages/detection-mixin.js

@@ -172,65 +172,5 @@ export default {
       this.detecting = false
       f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
     },
-    async videoFrameDetect (vidData) {
-      await this.loadModel(this.miniLocation)
-      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
-      const imCanvas = this.$refs.image_cvs
-      const imageCtx = imCanvas.getContext("2d")
-      const target = this.$refs.target_image
-      await tf.nextFrame();
-      imCanvas.width = imCanvas.clientWidth
-      imCanvas.height = imCanvas.clientHeight
-      imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-      var imgWidth
-      var imgHeight
-      const imgAspect = vidData.width / vidData.height
-      const rendAspect = imCanvas.width / imCanvas.height
-      if (imgAspect >= rendAspect) {
-        imgWidth = imCanvas.width
-        imgHeight = imCanvas.width / imgAspect
-      } else {
-        imgWidth = imCanvas.height * imgAspect
-        imgHeight = imCanvas.height
-      }
-      while (this.videoAvailable) {
-        console.time('frame-process')
-        try {
-          const input = tf.tidy(() => {
-            return tf.image.resizeBilinear(tf.browser.fromPixels(vidData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
-          })
-          const res = model.predict(input)
-          const rawRes = tf.transpose(res,[0,2,1]).arraySync()[0]
-
-          let rawCoords = []
-          if (rawRes) {
-            for (var i = 0; i < rawRes.length; i++) {
-              let getScores = rawRes[i].slice(4) 
-              if (getScores.some( s => s > .5)) {
-                let foundTarget = rawRes[i].slice(0,2)
-                foundTarget.push(Math.max(...getScores))
-                rawCoords.push(foundTarget)
-              }
-            }
-
-            imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
-            for (var coord of rawCoords) {
-              console.log(`x: ${coord[0]}, y: ${coord[1]}`)
-              let pointX = (imCanvas.width - imgWidth) / 2 + (coord[0] / modelWidth) * imgWidth -5
-              let pointY = (imCanvas.height - imgHeight) / 2 + (coord[1] / modelHeight) * imgHeight -5
-              imageCtx.globalAlpha = coord[2]
-              imageCtx.drawImage(target, pointX, pointY, 20, 20)
-            }
-          }
-          tf.dispose(input)
-          tf.dispose(res)
-          tf.dispose(rawRes)
-        } catch (e) {
-          console.log(e)
-        }
-        console.timeEnd('frame-process')
-        await tf.nextFrame();
-      }
-    }
   }
 }