Split detect.vue (#122)

Closes: #112

New mixins for camera and detection (remote and local) and new css for detection page.

Reviewed-on: #122

src/css/detect.css

@@ -0,0 +1,183 @@
+/*Styles for the structure detection page*/
+/*Basic style*/
+.detect-grid {
+  display: grid;
+  grid-template-columns: 1fr;
+  grid-template-rows: 1fr 56px auto min-content;
+  grid-template-areas: 
+    "image-view"
+    "result-view"
+    "detect-settings"
+    "menu-view";
+  justify-items: center;
+  height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
+  max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
+}
+
+.image-container {
+  grid-area: image-view;
+  width: 100%;
+  height: 100%;
+  min-width: 0;
+  min-height: 0;
+  position: relative;
+  display: flex;
+  align-self: stretch;
+}
+
+.popover-button-menu {
+  max-width: 90vw;
+  max-height: 90vh;
+  width: auto;
+}
+
+.segment-button-menu {
+  flex-wrap: nowrap;
+  flex-direction: column;
+  max-height: 100%;
+  min-height: 0px;
+}
+
+.chip-media {
+  background-color: var(--chip-media-background) !important;
+}
+
+.chip-results {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 5px;
+  padding: 10px;
+  --f7-chip-border-radius: 16px;
+  --f7-chip-media-size: 32px;
+  --f7-chip-font-weight: normal;
+  overflow-y: auto;
+  max-height: 100%;
+}
+
+.chip-results .chip {
+  padding-left: 8px;
+}
+
+.selected-chip {
+  font-weight: 500;
+  box-shadow: 4px 4px 1px var(--avn-theme-color);
+  transform: translate(-2px, -2px);
+}
+
+.detect-inputs {
+  display: flex;
+  align-items: center;
+  margin: 5px;
+  width: 100%;
+  max-width: 400px;
+  min-width: 192px;
+}
+
+.level-slide-vert {
+  display: none;
+}
+
+.image-menu {
+  grid-area: menu-view;
+  margin: 5px;
+  max-width: 400px;
+  min-width: 192px;
+}
+
+.image-menu .button {
+  aspect-ratio: 1;
+  height: auto;
+  padding: 5px;
+  flex: 1 1 0%;
+}
+
+.image-menu > .button > svg {
+  aspect-ratio: 1;
+  height: auto;
+  width: 100%;
+}
+
+.segment-button-menu .button {
+  padding: 8px;
+  aspect-ratio: 1;
+  width: auto;
+  flex: 1 1 0%;
+  max-height: 100px;
+  max-width: 100px;
+}
+
+/*Additional styles for small format landscape orientation*/
+@media (max-height: 450px) and (orientation: landscape) {
+  .detect-grid {
+    grid-template-columns: minmax(0,1fr) minmax(56px,max-content) auto auto;
+    grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
+    grid-template-areas: 
+      "image-view result-view detect-settings menu-view";
+    justify-items: stretch;
+    align-items: stretch;
+    height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
+    max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
+    position: relative;
+  }
+
+  .chip-results {
+    flex-direction: column;
+    max-height: 100%;
+    justify-self: start;
+    flex-wrap: nowrap;
+  }
+  
+  .detect-inputs {
+    flex-direction: column;
+    min-width: 0;
+    max-width: 72px;
+  }
+
+  .level-slide-horz {
+    display: none;
+  }
+
+  .level-slide-vert {
+    display: block;
+  }
+
+
+  .image-container {
+    flex-direction: column;
+  }
+
+  .image-menu {
+    flex-direction: column;
+    aspect-ratio: .25;
+    width: auto;
+    min-width: 0;
+    height: 100%;
+  }
+
+  .image-menu .button {
+    aspect-ratio: 1;
+    width: auto;
+    height: 100%;
+    flex: 1 1 0%;
+    border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
+    border-bottom-left-radius: 0px !important;
+  }
+
+  .segment-button-menu {
+    flex-direction: row;
+    max-height: 100%;
+    min-height: 0px;
+  }
+
+  .segment-button-menu .button {
+    height: auto;
+    flex: 1 1 0%;
+    max-height: 100px;
+    max-width: 100px;
+  }
+
+  .button > svg {
+    width: 100%;
+    height: auto;
+  }
+}

src/pages/camera-mixin.js

@@ -0,0 +1,45 @@
+export default {
+  methods: {
+    async openCamera() {
+      var cameraLoaded = false
+      const devicesList = await navigator.mediaDevices.enumerateDevices()
+      this.videoDeviceAvailable = devicesList.some( d => d.kind == "videoinput")
+      if (this.videoDeviceAvailable) {
+        navigator.mediaDevices.getUserMedia({video: true})
+        var vidConstraint = {
+          video: {
+            width: {
+              ideal: 1920
+            },
+            height: {
+              ideal: 1080
+            },
+            facingMode: 'environment'
+          },
+          audio: false
+        }
+        const stream = await navigator.mediaDevices.getUserMedia(vidConstraint);
+        cameraLoaded = true
+        this.cameraStream = stream
+      }
+      return cameraLoaded
+    },
+    closeCamera () {
+      this.cameraStream.getTracks().forEach( t => t.stop())
+      this.videoAvailable = false
+    },
+    captureVidFrame() {
+      const vidViewer = this.$refs.vid_viewer
+      vidViewer.pause()
+      let tempCVS = document.createElement('canvas')
+      tempCVS.height = vidViewer.videoHeight || parseInt(vidViewer.style.height)
+      tempCVS.width = vidViewer.videoWidth || parseInt(vidViewer.style.width)
+      const tempCtx = tempCVS.getContext('2d')
+      tempCtx.drawImage(vidViewer, 0, 0)
+      this.getImage(tempCVS.toDataURL())
+    },
+    async videoStream() {
+      //TODO
+    }
+  }
+}

src/pages/detect.vue

@@ -99,189 +99,7 @@
   </f7-page>
 </template>
 
-<style>
+<style src="../css/detect.css" />
-  .detect-grid {
-    display: grid;
-    grid-template-columns: 1fr;
-    grid-template-rows: 1fr 56px auto min-content;
-    grid-template-areas: 
-      "image-view"
-      "result-view"
-      "detect-settings"
-      "menu-view";
-    justify-items: center;
-    height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
-    max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
-  }
-
-  .image-container {
-    grid-area: image-view;
-    width: 100%;
-    height: 100%;
-    min-width: 0;
-    min-height: 0;
-    position: relative;
-    display: flex;
-    align-self: stretch;
-  }
-
-  .popover-button-menu {
-    max-width: 90vw;
-    max-height: 90vh;
-    width: auto;
-  }
-  
-  .segment-button-menu {
-    flex-wrap: nowrap;
-    flex-direction: column;
-    max-height: 100%;
-    min-height: 0px;
-  }
-
-  .chip-media {
-    background-color: var(--chip-media-background) !important;
-  }
-
-  .chip-results {
-    display: flex;
-    flex-wrap: wrap;
-    gap: 5px;
-    padding: 10px;
-    --f7-chip-border-radius: 16px;
-    --f7-chip-media-size: 32px;
-    --f7-chip-font-weight: normal;
-    overflow-y: auto;
-    max-height: 100%;
-  }
-
-  .chip-results .chip {
-    padding-left: 8px;
-  }
-
-  .selected-chip {
-    font-weight: 500;
-    box-shadow: 4px 4px 1px var(--avn-theme-color);
-    transform: translate(-2px, -2px);
-  }
-
-  .detect-inputs {
-    display: flex;
-    align-items: center;
-    margin: 5px;
-    width: 100%;
-    max-width: 400px;
-    min-width: 192px;
-  }
-
-  .level-slide-vert {
-    display: none;
-  }
-
-  .image-menu {
-    grid-area: menu-view;
-    margin: 5px;
-    /*width: 90%;*/
-    max-width: 400px;
-    min-width: 192px;
-  }
-
-  .image-menu .button {
-    aspect-ratio: 1;
-    height: auto;
-    padding: 5px;
-    flex: 1 1 0%;
-  }
-
-  .image-menu > .button > svg {
-    aspect-ratio: 1;
-    height: auto;
-    width: 100%;
-  }
-
-  .segment-button-menu .button {
-    padding: 8px;
-    aspect-ratio: 1;
-    width: auto;
-    flex: 1 1 0%;
-    max-height: 100px;
-    max-width: 100px;
-  }
-
-  @media (max-height: 450px) and (orientation: landscape) {
-    .detect-grid {
-      grid-template-columns: minmax(0,1fr) minmax(56px,max-content) auto auto;
-      grid-template-rows: calc(100vh - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom) - 64px);
-      grid-template-areas: 
-        "image-view result-view detect-settings menu-view";
-      justify-items: stretch;
-      align-items: stretch;
-      height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
-      max-height: calc(100% - var(--f7-navbar-height) - var(--f7-safe-area-top) - var(--f7-safe-area-bottom));
-      position: relative;
-    }
-
-    .chip-results {
-      flex-direction: column;
-      max-height: 100%;
-      justify-self: start;
-      flex-wrap: nowrap;
-    }
-    
-    .detect-inputs {
-      flex-direction: column;
-      min-width: 0;
-      max-width: 72px;
-    }
-
-    .level-slide-horz {
-      display: none;
-    }
-
-    .level-slide-vert {
-      display: block;
-    }
-
-
-    .image-container {
-      flex-direction: column;
-    }
-
-    .image-menu {
-      flex-direction: column;
-      aspect-ratio: .25;
-      width: auto;
-      min-width: 0;
-      height: 100%;
-    }
-
-    .image-menu .button {
-      aspect-ratio: 1;
-      width: auto;
-      height: 100%;
-      flex: 1 1 0%;
-      border-bottom: 1px solid var(--f7-segmented-raised-divider-color);
-      border-bottom-left-radius: 0px !important;
-    }
-  
-    .segment-button-menu {
-      flex-direction: row;
-      max-height: 100%;
-      min-height: 0px;
-    }
-
-    .segment-button-menu .button {
-      height: auto;
-      flex: 1 1 0%;
-      max-height: 100px;
-      max-width: 100px;
-    }
-
-    .button > svg {
-      width: 100%;
-      height: auto;
-    }
-  }
-</style>
 
 <script>
   import { f7 } from 'framework7-vue'
@@ -291,12 +109,13 @@
   import SvgIcon from '../components/svg-icon.vue'
 
   import submitMixin from './submit-mixin'
-  import detectMixin from './local-detect'
+  import detectionMixin from './detection-mixin'
+  import cameraMixin from './camera-mixin'
 
   import thoraxClasses from '../models/thorax/classes.json'
 
   export default {
-    mixins: [submitMixin, detectMixin],
+    mixins: [submitMixin, detectionMixin, cameraMixin],
     props: {
       f7route: Object,
     },
@@ -360,35 +179,13 @@
       }
       var loadServerSettings = localStorage.getItem('serverSettings')
       if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
-      var self = this
       if (this.serverSettings && this.serverSettings.use) {
-        var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
+        this.getRemoteLabels()
-        var xhr = new XMLHttpRequest()
-        xhr.open("GET", modelURL)
-        xhr.setRequestHeader('Content-Type', 'application/json')
-        xhr.timeout = 10000
-        xhr.ontimeout = this.remoteTimeout
-        xhr.onload = function () {
-          if (this.status !== 200) {
-            console.log(xhr.response)
-            const errorResponse = JSON.parse(xhr.response)
-            f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
-            return;
-          }
-          var detectors = JSON.parse(xhr.response).detectors
-          var findLabel = detectors
-            .find( d => { return d.name == self.detectorName } )?.labels
-            .filter( l => { return l != "" } ).sort()
-            .map( l => { return {'name': l, 'detect': true} } )
-          self.detectorLabels = findLabel || []
-        }
-        
-        xhr.send()
       } else {
-        self.modelLoading = true
+        this.modelLoading = true
-        self.detectorLabels = self.classesList.map( l => { return {'name': l, 'detect': true} } )
+        this.detectorLabels = this.classesList.map( l => { return {'name': l, 'detect': true} } )
-        self.loadModel(self.isCordova ? self.modelLocationCordova : self.modelLocation).then(() => {
+        this.loadModel(this.isCordova ? this.modelLocationCordova : this.modelLocation).then(() => {
-          self.modelLoading = false
+          this.modelLoading = false
         }).catch((e) => {
           console.log(e.message)
           f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
@@ -434,52 +231,21 @@
         return `--chip-media-background: hsl(${confVal / 100 * 120}deg 100% 50%)`
       },
       setData () {
-        var self = this
         if (this.serverSettings && this.serverSettings.use) {
-          var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
+          this.remoteDetect()
-          var xhr = new XMLHttpRequest()
-          xhr.open("POST", modelURL)
-          xhr.timeout = 10000
-          xhr.ontimeout = this.remoteTimeout
-          xhr.setRequestHeader('Content-Type', 'application/json')
-          xhr.onload = function () {
-            self.detecting = false
-            if (this.status !== 200) {
-              console.log(xhr.response)
-              const errorResponse = JSON.parse(xhr.response)
-              f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
-              return;
-            }
-            self.resultData = JSON.parse(xhr.response)
-            self.uploadDirty = true
-          }
-          
-          var doodsData = {
-            "detector_name": this.detectorName,
-            "detect": {
-              "*": 1
-            },
-            "data": this.imageView.src.split(',')[1]
-          }
-
-          xhr.send(JSON.stringify(doodsData))
         } else {
           this.localDetect(this.imageView).then(dets => {
-            self.detecting = false
+            this.detecting = false
-            self.resultData = dets
+            this.resultData = dets
-            self.uploadDirty = true
+            this.uploadDirty = true
           }).catch((e) => {
             console.log(e.message)
-            self.detecting = false
+            this.detecting = false
-            self.resultData = {}
+            this.resultData = {}
             f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
           })
         }
       },
-      remoteTimeout () {
-        this.detecting = false
-        f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
-      },
       selectAll (ev) {
         if (ev.target.checked) {
           this.detectorLabels.forEach( s => s.detect = true )
@@ -494,29 +260,10 @@
             return
         }
         if (mode == "camera") {
-          const devicesList = await navigator.mediaDevices.enumerateDevices()
+            this.videoAvailable = await this.openCamera()
-          this.videoDeviceAvailable = devicesList.some( d => d.kind == "videoinput")
+            if (this.videoAvailable) { return }
-          if (this.videoDeviceAvailable) {
-            navigator.mediaDevices.getUserMedia({video: true})
-            var vidConstraint = {
-              video: {
-                width: {
-                  ideal: 1920
-                },
-                height: {
-                  ideal: 1080
-                },
-                facingMode: 'environment'
-              },
-              audio: false
-            }
-            const stream = await navigator.mediaDevices.getUserMedia(vidConstraint);
-            this.videoAvailable = true
-            this.cameraStream = stream
-            return
-          }
         }
-        var loadResult = this.$refs.image_chooser.click()
+        this.$refs.image_chooser.click()
       },
       onFail (message) {
         alert(`Camera fail: ${message}`)
@@ -565,8 +312,7 @@
       getImage (searchImage) {
         let loadImage = new Promise(resolve => {
           if (this.videoAvailable) {
-            this.cameraStream.getTracks().forEach( t => t.stop())
+            this.closeCamera()
-            this.videoAvailable = false
             this.detecting = true
             resolve(searchImage)
           } else if (this.isCordova && this.imageLoadMode == "camera") {
@@ -604,19 +350,6 @@
           f7.dialog.alert(`Error loading image: ${e.message}`)
         })
       },
-      async videoStream() {
-        //TODO
-      },
-      captureVidFrame() {
-        const vidViewer = this.$refs.vid_viewer
-        vidViewer.pause()
-        let tempCVS = document.createElement('canvas')
-        tempCVS.height = vidViewer.videoHeight || parseInt(vidViewer.style.height)
-        tempCVS.width = vidViewer.videoWidth || parseInt(vidViewer.style.width)
-        const tempCtx = tempCVS.getContext('2d')
-        tempCtx.drawImage(vidViewer, 0, 0)
-        this.getImage(tempCVS.toDataURL())
-      },
       async submitData () {
         var uploadData = this.showResults
           .filter( d => { return d.aboveThreshold && d.isSearched && !d.isDeleted })

src/pages/detection-mixin.js

@@ -1,4 +1,5 @@
 import * as tf  from '@tensorflow/tfjs'
+import { f7 } from 'framework7-vue'
 
 var model = null
 
@@ -77,6 +78,70 @@ export default {
       console.timeEnd('post-process')
 
       return output
+    },
+    getRemoteLabels() {
+      var self = this
+      var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detectors`
+      var xhr = new XMLHttpRequest()
+      xhr.open("GET", modelURL)
+      xhr.setRequestHeader('Content-Type', 'application/json')
+      xhr.timeout = 10000
+      xhr.ontimeout = this.remoteTimeout
+      xhr.onload = function () {
+        if (this.status !== 200) {
+          console.log(xhr.response)
+          const errorResponse = JSON.parse(xhr.response)
+          f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
+          return
+        }
+        var detectors = JSON.parse(xhr.response).detectors
+        var findLabel = detectors
+          .find( d => { return d.name == self.detectorName } )?.labels
+          .filter( l => { return l != "" } ).sort()
+          .map( l => { return {'name': l, 'detect': true} } )
+        self.detectorLabels = findLabel || []
+      }
+      xhr.onerror = function (e) {
+        f7.dialog.alert('ALVINN has encountered an unknown server error')
+        return
+      }
+      
+      xhr.send()
+    },
+    remoteDetect() {
+      var self = this
+      var modelURL = `http://${this.serverSettings.address}:${this.serverSettings.port}/detect`
+      var xhr = new XMLHttpRequest()
+      xhr.open("POST", modelURL)
+      xhr.timeout = 10000
+      xhr.ontimeout = this.remoteTimeout
+      xhr.setRequestHeader('Content-Type', 'application/json')
+      xhr.onload = function () {
+        self.detecting = false
+        if (this.status !== 200) {
+          console.log(xhr.response)
+          const errorResponse = JSON.parse(xhr.response)
+          f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
+          return;
+        }
+        self.resultData = JSON.parse(xhr.response)
+        self.uploadDirty = true
+      }
+      
+      var doodsData = {
+        "detector_name": this.detectorName,
+        "detect": {
+          "*": 1
+        },
+        "data": this.imageView.src.split(',')[1]
+      }
+
+      xhr.send(JSON.stringify(doodsData))
+    },
+    remoteTimeout () {
+      this.detecting = false
+      f7.dialog.alert('No connection to remote ALVINN instance.  Please check app settings.')
     }
+
   }
 }