Release 0.3.0

Signed-off-by: Justin Georgi <justin.georgi@gmail.com>

cordova/config.xml

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<widget id="edu.midwestern.alvinn" version="0.2.0" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
+<widget id="edu.midwestern.alvinn" version="0.3.0" xmlns="http://www.w3.org/ns/widgets" xmlns:cdv="http://cordova.apache.org/ns/1.0" xmlns:android="http://schemas.android.com/apk/res/android">
     <name>ALVINN</name>
     <description>Anatomy Lab Visual Identification Neural Network.</description>
     <author email="jgeorg@midwestern.edu" href="https://midwestern.edu">

cordova/package.json

@@ -1,7 +1,7 @@
 {
   "name": "edu.midwestern.alvinn",
   "displayName": "ALVINN",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Anatomy Lab Visual Identification Neural Network.",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "alvinn",
   "private": true,
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "ALVINN",
   "repository": "",
   "license": "UNLICENSED",
@@ -23,6 +23,7 @@
     "last 5 Firefox versions"
   ],
   "dependencies": {
+    "@tensorflow/tfjs": "^4.17.0",
     "dom7": "^4.0.6",
     "framework7": "^8.3.0",
     "framework7-icons": "^5.0.5",

src/components/app.vue

@@ -7,10 +7,11 @@
         <f7-navbar title="ALVINN"></f7-navbar>
         <f7-list>
           <f7-list-item link="/settings/" view=".view-main" panel-close=".panel-left">Settings</f7-list-item>
-          <f7-list-item link="/about/">About ALVINN</f7-list-item>
+          <f7-list-item link="/about/" >About ALVINN</f7-list-item>
+          <f7-list-item link="/contact/" view=".view-main" panel-close=".panel-left">Contact</f7-list-item>
         </f7-list>
         <f7-toolbar class="panel-bar" position="bottom">
-          <span>version 0.2.0</span>
+          <span>version 0.3.0</span>
         </f7-toolbar>
       </f7-page>
     </f7-view>

src/js/routes.js

@@ -3,6 +3,7 @@ import HomePage from '../pages/home.vue';
 import AboutPage from '../pages/about.vue';
 import SettingsPage from '../pages/settings.vue';
 import DetectPage from '../pages/detect.vue';
+import ContactPage from  '../pages/contact.vue';
 
 import NotFoundPage from '../pages/404.vue';
 
@@ -23,6 +24,10 @@ var routes = [
     path: '/settings/',
     component: SettingsPage,
   },
+  {
+    path: '/contact/',
+    component: ContactPage,
+  },
   {
     path: '(.*)',
     component: NotFoundPage,

src/js/store.js

@@ -1,7 +1,8 @@
 import { reactive, computed } from 'vue';
 
 const state = reactive({
-    disclaimerAgreement: false
+    disclaimerAgreement: false,
+    enabledRegions: ['thorax','abdomen','limbs']
   })
 
 const agree = () => {
@@ -10,5 +11,6 @@ const agree = () => {
 
 export default () => ({
   isAgreed: computed(() => state.disclaimerAgreement),
+  getRegions: computed(() => state.enabledRegions),
   agree
 })

src/models/thorax_tfwm/classes.json

@@ -0,0 +1,12 @@
+[
+  "Right lung",
+  "Diaphragm",
+  "Heart",
+  "Caudal vena cava",
+  "Cranial vena cava",
+  "Phrenic nerve",
+  "Trachea",
+  "Vagus nerve",
+  "Left Lung",
+  "Aorta"
+]

src/pages/contact.vue

@@ -0,0 +1,95 @@
+<template>
+  <f7-page name="contact">
+    <f7-navbar title="Contact" back-link="Back"></f7-navbar>
+    <f7-block-title medium style="text-align: center;">Contact the ALVINN team</f7-block-title>
+    <f7-block style="display: flex; justify-content: center;">
+      <div class="form-container"> 
+        <p>
+          ALVINN can only get better with your feedback.  Use the form below to send us any questions or let us know how we can improve.
+        </p>
+        <f7-list class="form-element">
+          <f7-list-input v-model:value="userEmail" label="E-mail (optional)" type="email" placeholder="Your e-mail" clear-button />
+          <f7-list-input v-model:value="commentType" label="This is a... (optional)" type="select" placeholder="Select comment type">
+            <option value="20">Bug</option>
+            <option value="22">Feature request</option>
+            <option value="25">Question</option>
+          </f7-list-input>
+          <f7-list-input v-model:value="commentTitle" label="Subject of comment (optional)" type="textarea" resizable placeholder="Type here" />
+        </f7-list>
+        <f7-text-editor class="form-element comment-editor"/>
+        <div style="align-self: flex-end; display: flex; gap: 15px;">
+          <f7-button fill @click="clearForm">
+            Clear form
+          </f7-button>
+          <f7-button fill @click="sendFeedback">
+            Send feedback
+          </f7-button>
+        </div>
+      </div>
+    </f7-block>
+  </f7-page>
+</template>
+
+<style>
+  .form-container {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: center;
+  }
+
+  .form-element {
+    align-self: stretch;
+  }
+</style>
+
+<script>
+  import { f7 } from 'framework7-vue'
+
+  export default {
+    data () {
+      return {
+        commentTitle: "",
+        userEmail: "",
+        commentType: ""
+      }
+    },
+    computed: {
+      commentText () {
+        var text = f7.textEditor.get('.comment-editor').getValue()
+        if (this.userEmail) {
+          text += `\\n\\nSubmitted by: ${this.userEmail}`
+        }
+        return text
+      }
+    },
+    methods: {
+      sendFeedback () {
+        var self = this
+        var issueURL = `https://gitea.azgeorgis.net/api/v1/repos/Georgi_Lab/ALVINN_f7/issues?access_token=9af8ae15b1ee5a98afcb3083bb488e4cf3c683af`
+        var xhr = new XMLHttpRequest()
+        xhr.open("POST", issueURL)
+        xhr.setRequestHeader('Content-Type', 'application/json')
+        xhr.setRequestHeader('accept', 'application/json')
+        xhr.onload = function () {
+          if (this.status !== 201) {
+            console.log(xhr.response)
+            const errorResponse = JSON.parse(xhr.response)
+            f7.dialog.alert(`ALVINN has encountered an error: ${errorResponse.error}`)
+            return;
+          }
+          f7.dialog.alert('Thank you for your feedback.')
+          self.clearForm()
+        }
+        xhr.send(`{"body": "${this.commentText}", "labels": [${this.commentType}], "title": "${this.commentTitle || 'User submitted comment'}"}`)
+      },
+      clearForm () {
+        this.commentTitle = ''
+        this.userEmail = ''
+        this.commentType = ''
+        f7.textEditor.get('.comment-editor').clearValue();
+      }
+    }
+  }
+
+</script>

src/pages/detect.vue

@@ -21,7 +21,7 @@
           :style="chipGradient(result.confidence)"
         />
         <span v-if="numResults == 0 && !detecting" style="height: var(--f7-chip-height); font-size: calc(var(--f7-chip-height) - 4px); font-weight: bolder; margin: 2px;">No results.</span>
-        <f7-preloader v-if="detecting" size="32" style="color: var(--avn-theme-color);" />
+        <f7-preloader v-if="detecting || modelLoading" size="32" style="color: var(--avn-theme-color);" />
       </div>
       <div v-if="showDetectSettings" class="detect-inputs" style="grid-area: detect-settings;">
         <f7-range class="level-slide-horz" :min="0" :max="100" :step="1" @range:change="onLevelChange" v-model:value="detectorLevel" type="range" style="flex: 1 1 100%"/>
@@ -63,16 +63,16 @@
 
     <f7-popover id="region-popover" class="popover-button-menu">
       <f7-segmented raised class="segment-button-menu">
-        <f7-button style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
+        <f7-button :class="(getRegions.includes('thorax')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
           <RegionIcon :region="0" />
         </f7-button>
-        <f7-button style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
+        <f7-button :class="(getRegions.includes('abdomen')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
           <RegionIcon :region="1" />
         </f7-button>
-        <f7-button style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
+        <f7-button :class="(getRegions.includes('limbs')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
           <RegionIcon :region="2" />
         </f7-button>
-        <f7-button class="disabled" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
+        <f7-button :class="(getRegions.includes('head')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
           <RegionIcon :region="3" />
         </f7-button>
       </f7-segmented>
@@ -146,6 +146,8 @@
     --f7-chip-border-radius: 16px;
     --f7-chip-media-size: 32px;
     --f7-chip-font-weight: normal;
+    overflow-y: auto;
+    max-height: 100%;
   }
 
   .chip-results .chip {
@@ -219,7 +221,6 @@
       max-height: 100%;
       justify-self: start;
       flex-wrap: nowrap;
-      overflow-y: scroll;
     }
     
     .detect-inputs {
@@ -281,13 +282,17 @@
 <script>
   import { f7 } from 'framework7-vue'
 
+  import store from '../js/store'
   import RegionIcon from '../components/region-icon.vue'
   import SvgIcon from '../components/svg-icon.vue'
 
   import submitMixin from './submit-mixin'
+  import detectMixin from './local-detect'
+
+  import thoraxClasses from '../models/thorax_tfwm/classes.json'
 
   export default {
-    mixins: [submitMixin],
+    mixins: [submitMixin, detectMixin],
     props: {
       f7route: Object,
     },
@@ -301,6 +306,7 @@
         resultData: {},
         selectedChip: -1,
         activeRegion: 4,
+        classesList: [],
         imageLoaded: false,
         imageView: null,
         imageLoadMode: "environment",
@@ -313,14 +319,25 @@
         serverSettings: {},
         isCordova: !!window.cordova,
         uploadUid: null,
-        uploadDirty: false
+        uploadDirty: false,
+        modelLocation: '',
+        modelLoading: false
       }
     },
+    setup() {
+      return store()
+    },
     created () {
       switch (this.f7route.params.region) {
         case 'thorax':
           this.activeRegion = 0
           this.detectorName = 'thorax'
+          this.classesList = thoraxClasses
+          /* VITE setting */
+          this.modelLocation = '../models/thorax_tfwm/model.json'
+          /* PWA Build setting */
+          //this.modelLocation = './models/thorax_tfwm/model.json'
+          this.modelLocationCordova = 'https://localhost/models/thorax_tfwm/model.json'
           break;
         case 'abdomen':
           this.activeRegion = 1
@@ -360,6 +377,15 @@
         }
         
         xhr.send()
+      } else {
+        self.modelLoading = true
+        self.detectorLabels = self.classesList.map( l => { return {'name': l, 'detect': true} } )
+        self.loadModel(self.isCordova ? self.modelLocationCordova : self.modelLocation).then(() => {
+          self.modelLoading = false
+        }).catch((e) => {
+          console.log(e.message)
+          f7.dialog.alert(`ALVINN AI model error: ${e.message}`)
+        })
       }
       window.onresize = (e) => { this.selectChip('redraw') }
     },
@@ -377,6 +403,10 @@
           filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel
           filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label)
         })
+
+        if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) {
+          this.selectChip(this.selectedChip)
+        }
         return filteredResults
       },
       numResults () {
@@ -427,8 +457,14 @@
 
           xhr.send(JSON.stringify(doodsData))
         } else {
-          //TODO
+          this.localDetect(this.imageView).then(dets => {
-          f7.dialog.alert('Using built-in model')
+            self.detecting = false
+            self.resultData = dets
+            self.uploadDirty = true
+          }).catch((e) => {
+          console.log(e.message)
+          f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
+        })
         }
       },
       remoteTimeout () {
@@ -522,7 +558,15 @@
         }).then( () => {
           const [imCanvas, _]  = this.resetView()
           imCanvas.style['background-image'] = `url(${this.imageView.src})`
-          this.setData()
+          /******
+           * setTimeout is not a good solution,
+           * but it's the only way I can find to
+           * not cut off drawing of of the progress
+           * spinner
+           ******/
+          setTimeout(() => {
+            this.setData()
+          }, 250)
         }).catch((e) => {
           console.log(e.message)
           f7.dialog.alert(`Error loading image: ${e.message}`)

src/pages/home.vue

@@ -13,20 +13,16 @@
       <h4 style="text-align: center; margin: 0;">Veterinary Anatomy Edition</h4>
       <p style="text-align: center; margin: 0;">Select a region to begin.</p>
       <div class="region-grid">
-        <!--</f7-button><f7-button :class="`region-button thorax`" :href="'/detect/thorax/'">-->
+        <f7-button :class="`region-button thorax${isAgreed && getRegions.includes('thorax') ? '' : ' disabled'}`" :href="isAgreed && getRegions.includes('thorax') && '/detect/thorax/'">
-        <f7-button :class="`region-button thorax${isAgreed ? '' : ' disabled'}`" :href="isAgreed && '/detect/thorax/'">
           <RegionIcon class="region-image" :region="0" />
         </f7-button>
-        <!--<f7-button :class="`region-button abdomen${siteSettings.siteAgreement ? '' : ' disabled'}`" :href="siteSettings.siteAgreement && '/detect/abdomen/'">-->
+        <f7-button :class="`region-button abdomen${isAgreed && getRegions.includes('abdomen') ? '' : ' disabled'}`" :href="isAgreed && getRegions.includes('abdomen') && '/detect/abdomen/'">
-        <f7-button :class="`region-button abdomen${isAgreed ? '' : ' disabled'}`" :href="isAgreed && '/detect/abdomen/'">
           <RegionIcon class="region-image" :region="1" />
         </f7-button>
-        <!--<f7-button :class="`region-button limbs${siteSettings.siteAgreement ? '' : ' disabled'}`" :href="siteSettings.siteAgreement && '/detect/limbs/'">-->
+        <f7-button :class="`region-button limbs${isAgreed && getRegions.includes('limbs') ? '' : ' disabled'}`" :href="isAgreed && getRegions.includes('limbs') && '/detect/limbs/'">
-        <f7-button :class="`region-button limbs${isAgreed ? '' : ' disabled'}`" :href="isAgreed && '/detect/limbs/'">
           <RegionIcon class="region-image" :region="2" />
         </f7-button>
-        <!--<f7-button class="region-button headneck disabled" :href="siteSettings.siteAgreement && '/detect/head/'">-->
+        <f7-button :class="`region-button headneck${isAgreed && getRegions.includes('head') ? '' : ' disabled'}`" :href="isAgreed && getRegions.includes('head') && '/detect/head/'">
-        <f7-button class="region-button headneck disabled" :href="'/detect/head/'">
           <RegionIcon class="region-image" :region="3" />
         </f7-button>
       </div>

src/pages/local-detect.js

@@ -0,0 +1,48 @@
+import * as tf  from '@tensorflow/tfjs'
+
+var model = null
+
+export default {
+  methods: {
+    async loadModel(weights) {
+      model = await tf.loadGraphModel(weights).then(graphModel => {
+        return graphModel
+      })
+    },
+    async localDetect(imageData) {
+      const [modelWidth, modelHeight] = model.inputs[0].shape.slice(1, 3);
+
+      const input = tf.tidy(() => {
+        return tf.image.resizeBilinear(tf.browser.fromPixels(imageData), [modelWidth, modelHeight]).div(255.0).expandDims(0)
+      })
+      var results = model.executeAsync(input).then(res => {
+        const [boxes, scores, classes, valid_detections] = res;
+        const boxes_data = boxes.dataSync();
+        const scores_data = scores.dataSync();
+        const classes_data = classes.dataSync();
+        const valid_detections_data = valid_detections.dataSync()[0];
+
+        tf.dispose(res)
+
+        var output = {
+          detections: []
+        }
+        for (var i =0; i < valid_detections_data; i++) {
+          var [dLeft, dTop, dRight, dBottom] = boxes_data.slice(i * 4, (i + 1) * 4);
+          output.detections.push({
+            "top": dTop,
+            "left": dLeft,
+            "bottom": dBottom,
+            "right": dRight,
+            "label": this.detectorLabels[classes_data[i]].name,
+            "confidence": scores_data[i] * 100
+          })
+        }
+
+        return output
+      })
+
+      return results
+    }
+  }
+}