Update thorax model and improve model performance (#125)

Closes: #117

This bumps the thorax model from the yolo nano to the yolo sm (64 x 640 size) but greatly improves model performance by running a fake detection event on page load to get the model parameters in memory.

As a result of that change a new loading message was required so the f7 preloader was switched out for an f7 progress bar and more messaging was added during various stages.  The progress bar fixes the previous issue with the preloader.

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

Reviewed-on: #125

src/css/detect.css

@@ -3,7 +3,7 @@
 .detect-grid {
   display: grid;
   grid-template-columns: 1fr;
-  grid-template-rows: 1fr 56px auto min-content;
+  grid-template-rows: 1fr minmax(calc(var(--f7-chip-height) + 33px), auto) auto min-content;
   grid-template-areas: 
     "image-view"
     "result-view"
@@ -51,6 +51,7 @@
   --f7-chip-media-size: 32px;
   --f7-chip-font-weight: normal;
   overflow-y: auto;
+  overflow-x: hidden;
   max-height: 100%;
 }
 
@@ -58,6 +59,10 @@
   padding-left: 8px;
 }
 
+.progress-hide {
+  display: hidden;
+}
+
 .selected-chip {
   font-weight: 500;
   box-shadow: 4px 4px 1px var(--avn-theme-color);

src/models/thorax/metadata.yaml

@@ -1,14 +1,14 @@
 description: Ultralytics best model trained on /data/ALVINN/Thorax/Thorax 0.1.0/thorax.yaml
 author: Ultralytics
 license: AGPL-3.0 https://ultralytics.com/license
-date: '2024-03-05T17:01:14.129654'
+date: '2024-03-07T16:03:03.296997'
 version: 8.1.20
 stride: 32
 task: detect
 batch: 1
 imgsz:
-- 992
-- 992
+- 640
+- 640
 names:
   0: Abdominal diaphragm
   1: Aorta

src/pages/detect.vue

@@ -13,7 +13,7 @@
           <f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%);" fill>Capture</f7-button>
         </div>
       </div>
-      <div v-if="(resultData && resultData.detections) || detecting" class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
+      <div class="chip-results" style="grid-area: result-view; flex: 0 0 auto; align-self: center;">
         <f7-chip v-for="result in showResults.filter( r => { return r.aboveThreshold && r.isSearched && !r.isDeleted })"
           :class="(result.resultIndex == selectedChip) ? 'selected-chip' : ''"
           :text="result.label"
@@ -24,8 +24,8 @@
           @delete="deleteChip(result.resultIndex)"
           :style="chipGradient(result.confidence)"
         />
-        <span v-if="numResults == 0 && !detecting" style="height: var(--f7-chip-height); font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">No results.</span>
-        <f7-preloader v-if="detecting || modelLoading" size="32" style="color: var(--avn-theme-color);" />
+        <div v-if="!numResults" style="height: var(--f7-chip-height); width: 100%; text-align: center; font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">{{  message }}</div>
+        <f7-progressbar v-if="(detecting || modelLoading)" style="width: 100%;" :infinite="true" />
       </div>
       <div v-if="showDetectSettings" class="detect-inputs" style="grid-area: detect-settings;">
         <f7-range class="level-slide-horz" :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
@@ -41,7 +41,7 @@
         <f7-button popover-open="#region-popover">
           <RegionIcon :region="activeRegion" />
         </f7-button>
-        <f7-button popover-open="#capture-popover">
+        <f7-button :class="(!modelLoading) ? '' : 'disabled'" popover-open="#capture-popover">
           <SvgIcon icon="camera_add"/>
         </f7-button>
         <f7-button @click="() => showDetectSettings = !showDetectSettings" :class="(imageLoaded) ? '' : 'disabled'">
@@ -144,7 +144,7 @@
         uploadUid: null,
         uploadDirty: false,
         modelLocation: '',
-        modelLoading: false,
+        modelLoading: true,
         videoDeviceAvailable: false,
         videoAvailable: false,
         cameraStream: null
@@ -179,8 +179,11 @@
       }
       var loadServerSettings = localStorage.getItem('serverSettings')
       if (loadServerSettings) this.serverSettings = JSON.parse(loadServerSettings)
+    },
+    mounted () {
       if (this.serverSettings && this.serverSettings.use) {
         this.getRemoteLabels()
+        this.modelLoading = false
       } else {
         this.modelLoading = true
         this.detectorLabels = this.classesList.map( l => { return {'name': l, 'detect': true} } )
@@ -189,11 +192,23 @@
         }).catch((e) => {
           console.log(e.message)
           f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
+          this.modelLoading = false
         })
       }
       window.onresize = (e) => { this.selectChip('redraw') }
     },
     computed: {
+      message () {
+        if (this.modelLoading) {
+          return "Preparing ALVINN..."
+        } else if (this.detecting) {
+          return "Finding structures..."
+        } else if (this.numResults == 0 && this.imageLoaded) {
+          return "No results."
+        } else {
+          return "ALVINN is ready."
+        }
+      },
       showResults () {
         var filteredResults = this.resultData.detections
         if (!filteredResults) return []

src/pages/detection-mixin.js

@@ -1,18 +1,22 @@
 import * as tf  from '@tensorflow/tfjs'
 import { f7 } from 'framework7-vue'
+import { nextTick } from 'vue'
 
 var model = null
 
 export default {
   methods: {
     async loadModel(weights) {
-      model = await tf.loadGraphModel(weights).then(graphModel => {
-        return graphModel
-      })
-  },
+      await nextTick()
+      model = await tf.loadGraphModel(weights)
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
+      const dummyT = tf.ones([1,modelWidth,modelHeight,3])
+      model.predict(dummyT) //Run model once to preload weights for better response time
+      return model
+    },
     async localDetect(imageData) {
       console.time('pre-process')
-      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3);
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3)
       const input = tf.tidy(() => {
         return tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
       })