Add Structure Class (#203)

This PR adds a class for detected structures and two additional classes to handle the coordinates of the bounding boxes for those structures.  The classes do a better job handling conversions between image pixels, canvas pixels, and screen pixels.  This means that they take the place of several chunks of the previous code such as the box2cvs function.

Reviewed-on: #203

src/js/structures.js

@@ -0,0 +1,157 @@
+class Coordinate {
+  constructor(x, y) {
+    this.x = x
+    this.y = y
+  }
+
+  toRefFrame(...frameArgs) {
+    if (frameArgs.length == 0) {
+      return {x: this.x, y: this.y}
+    }
+    let outFrames = []
+    //Get Coordinates in Image Reference Frame
+    if (frameArgs[0].tagName == 'IMG' && frameArgs[0].width && frameArgs[0].height) {
+      outFrames.push({
+        x: this.x * frameArgs[0].width,
+        y: this.y * frameArgs[0].height
+      })
+    } else {
+      throw new Error('Coordinate: invalid reference frame for frameType: Image')
+    }
+    //Get Coordinates in Canvas Reference Frame
+    if (frameArgs[1]) {
+      if (frameArgs[1].tagName == 'CANVAS' && frameArgs[1].width && frameArgs[1].height) {
+        let imgWidth
+        let imgHeight
+        const imgAspect = frameArgs[0].width / frameArgs[0].height
+        const rendAspect = frameArgs[1].width / frameArgs[1].height
+        if (imgAspect >= rendAspect) {
+          imgWidth = frameArgs[1].width
+          imgHeight = frameArgs[1].width / imgAspect
+        } else {
+          imgWidth = frameArgs[1].height * imgAspect
+          imgHeight = frameArgs[1].height
+        }
+        outFrames.push({
+          x: (frameArgs[1].width - imgWidth) / 2 + this.x * imgWidth,
+          y: (frameArgs[1].height - imgHeight) / 2 + this.y * imgHeight
+        })
+      } else {
+        throw new Error('Coordinate: invalid reference frame for frameType: Canvas')
+      }
+    }
+    //Get Coordinates in Screen Reference Frame
+    if (frameArgs[2]) {
+      if (frameArgs[2].zoom && frameArgs[2].offset && frameArgs[2].offset.x !== undefined && frameArgs[2].offset.y !== undefined) {
+        outFrames.push({
+          x: outFrames[1].x * frameArgs[2].zoom + frameArgs[2].offset.x,
+          y: outFrames[1].y  * frameArgs[2].zoom + frameArgs[2].offset.y
+        })
+      } else {
+        throw new Error('Coordinate: invalid reference frame for frameType: Screen')
+      }
+    }
+
+    return outFrames
+  }
+
+  toString() {
+    return `(x: ${this.x}, y: ${this.y})`
+  }
+}
+
+export class StructureBox {
+  constructor(top, left, bottom, right) {
+    this.topLeft = new Coordinate(left, top)
+    this.bottomRight = new Coordinate(right, bottom)
+  }
+
+  getBoxes(boxType, ...frameArgs) {
+    let lowerH, lowerV, calcSide
+    switch (boxType) {
+      case 'point':
+        lowerH = 'right'
+        lowerV = 'bottom'
+        break
+      case 'side':
+        lowerH = 'width'
+        lowerV = 'height'
+        calcSide = true
+        break
+      default:
+        throw new Error(`StructureBox: invalid boxType - ${boxType}`)
+    }
+    if (frameArgs.length == 0) {
+      return {
+        left: this.topLeft.x,
+        top: this.topLeft.y,
+        [lowerH]: this.bottomRight.x - ((calcSide) ? this.topLeft.x : 0),
+        [lowerV]: this.bottomRight.y - ((calcSide) ? this.topLeft.y : 0)
+      }  
+    }
+    const tL = this.topLeft.toRefFrame(...frameArgs)
+    const bR = this.bottomRight.toRefFrame(...frameArgs)
+    let outBoxes = []
+    tL.forEach((cd, i) => {
+      outBoxes.push({
+        left: cd.x,
+        top: cd.y,
+        [lowerH]: bR[i].x - ((calcSide) ? cd.x : 0),
+        [lowerV]: bR[i].y - ((calcSide) ? cd.y : 0)
+      })
+    })
+    return outBoxes
+  }
+}
+
+export class Structure {
+  constructor(structResult) {
+    this.label = structResult.label
+    this.confidence = structResult.confidence
+    this.box = new StructureBox(
+      structResult.top,
+      structResult.left,
+      structResult.bottom,
+      structResult.right
+    )
+    this.deleted = false
+    this.index = -1
+    this.passThreshold = true
+    this.searched = false
+  }
+
+  get resultIndex() {
+    return this.index
+  }
+
+  set resultIndex(newIdx) {
+    this.index = newIdx
+  }
+
+  get isDeleted() {
+    return this.deleted
+  }
+
+  set isDeleted(del) {
+    this.deleted = !!del
+  }
+
+  get isSearched() {
+    return this.searched
+  }
+
+  set isSearched(ser) {
+    this.searched = !!ser
+  }
+
+  get aboveThreshold() {
+    return this.passThreshold
+  }
+
+  setThreshold(level) {
+    if (typeof level != 'number') {
+      throw new Error(`Structure: invalid threshold level ${level}`)
+    }
+    this.passThreshold = this.confidence >= level
+  }
+}

src/pages/detect.vue

@@ -144,11 +144,11 @@
   import touchMixin from './touch-mixin'
 
   import detectionWorker from '@/assets/detect-worker.js?worker&inline'
+  import { Structure, StructureBox } from '../js/structures'
 
   const regions = ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck']
   let activeRegion = 4
   let classesList = []
-  let imCvsLocation = {}
   let imageLoadMode = "environment"
   let serverSettings = {}
   let otherSettings = {}
@@ -158,6 +158,7 @@
   let detectWorker = null
   let vidWorker = null
   let canvasMoving = false
+  let imageLocation = new StructureBox(0, 0, 1, 1)
   
   export default {
     mixins: [submitMixin, detectionMixin, cameraMixin, touchMixin],
@@ -286,14 +287,14 @@
         var filteredResults = this.resultData.detections
         if (!filteredResults) return []
 
-        var allSelect = this.detectorLabels.every( s => { return s.detect } )
+        const allSelect = this.detectorLabels.every( s => { return s.detect } )
-        var selectedLabels = this.detectorLabels
+        const selectedLabels = this.detectorLabels
             .filter( l => { return l.detect })
             .map( l => { return l.name })
         filteredResults.forEach( (d, i) => {
-          filteredResults[i].resultIndex = i
+          d.resultIndex = i
-          filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel
+          d.setThreshold(this.detectorLevel)
-          filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label)
+          d.isSearched = allSelect || selectedLabels.includes(d.label)
         })
 
         if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) {
@@ -333,15 +334,17 @@
             self = this
             if (eDetect.data.error) {
               self.detecting = false
-              self.this.resultData = {}
+              self.resultData = {}
               loadFailure()
               f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`)
             } else if (eDetect.data.success == 'detection') {
               self.detecting = false
-              self.resultData = eDetect.data.detections
+              self.resultData = {detections: []}
-              if (self.resultData) {
+              eDetect.data.detections.detections.forEach((d) => {
-                self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name})
+                d.label = self.detectorLabels[d.label].name
-              }
+                let detectedStructure = new Structure(d)
+                self.resultData.detections.push(detectedStructure)
+              })
               self.uploadDirty = true
             } else if (eDetect.data.success == 'model') {
               reloadModel = false
@@ -476,33 +479,25 @@
           iChip = this.selectedChip
         }
         const [imCanvas, imageCtx] = this.resetView(true)
-        let structLeft = this.imageView.width * this.resultData.detections[iChip].left
+        let structBox, cvsBox, screenBox
-        let structTop = this.imageView.height * this.resultData.detections[iChip].top
+        [structBox, cvsBox, screenBox] = this.resultData.detections[iChip].box.getBoxes('side', this.imageView, imCanvas, {zoom: this.canvasZoom, offset: {...this.canvasOffset}})
-        let structWidth = this.imageView.width * (this.resultData.detections[iChip].right - this.resultData.detections[iChip].left)
-        let structHeight = this.imageView.height * (this.resultData.detections[iChip].bottom - this.resultData.detections[iChip].top)
 
-        const boxCoords = this.box2cvs(this.resultData.detections[iChip])[0]
+        this.infoLinkPos.x = Math.min(Math.max(screenBox.left, 0),imCanvas.width)
+        this.infoLinkPos.y = Math.min(Math.max(screenBox.top, 0), imCanvas.height)
 
-        let boxLeft = boxCoords.cvsLeft
+        const imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height)
-        let boxTop = boxCoords.cvsTop
+        imageCtx.drawImage(this.imageView, structBox.left, structBox.top, structBox.width, structBox.height, cvsBox.left, cvsBox.top, cvsBox.width, cvsBox.height)
-        let boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
-        let boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
-        this.infoLinkPos.x = Math.min(Math.max(boxCoords.cvsLeft * this.canvasZoom + this.canvasOffset.x, 0),imCanvas.width)
-        this.infoLinkPos.y = Math.min(Math.max(boxCoords.cvsTop * this.canvasZoom + this.canvasOffset.y, 0), imCanvas.height)
-
-        let imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height)
-        imageCtx.drawImage(this.imageView, structLeft, structTop, structWidth, structHeight, boxLeft, boxTop, boxWidth, boxHeight)
         imageCtx.save()
-        imageCtx.arc(boxLeft, boxTop, 14 / this.canvasZoom, 0, 2 * Math.PI)
+        imageCtx.arc(cvsBox.left, cvsBox.top, 14 / this.canvasZoom, 0, 2 * Math.PI)
         imageCtx.closePath()
         imageCtx.clip()
         imageCtx.drawImage(this.imageView,
-          structLeft - (14 / this.canvasZoom * imageScale),
+          structBox.left - (14 / this.canvasZoom * imageScale),
-          structTop - (14 / this.canvasZoom * imageScale),
+          structBox.top - (14 / this.canvasZoom * imageScale),
           (28 / this.canvasZoom * imageScale),
           (28 / this.canvasZoom * imageScale),
-          boxLeft - (14 / this.canvasZoom),
+          cvsBox.left - (14 / this.canvasZoom),
-          boxTop - (14 / this.canvasZoom),
+          cvsBox.top - (14 / this.canvasZoom),
           (28 / this.canvasZoom), (28 / this.canvasZoom))
         imageCtx.restore()
         this.selectedChip = iChip
@@ -532,13 +527,9 @@
         imageCtx.strokeStyle = 'yellow'
         imageCtx.lineWidth = 3 / this.canvasZoom
         if (this.imageLoaded) {
-          let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1})
+          const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas)
-          imCvsLocation.top = imageLoc[0].cvsTop
-          imCvsLocation.left = imageLoc[0].cvsLeft
-          imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft
-          imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop
           if (drawChip) {imageCtx.globalAlpha = .5}
-          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height)
+          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height)
           if (drawChip) {imageCtx.globalAlpha = 1}
         }
         this.structureZoomed = false
@@ -592,12 +583,8 @@
           imCanvas.width = imCanvas.clientWidth
           imCanvas.height = imCanvas.clientHeight
           const imageCtx = imCanvas.getContext("2d")
-          let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1})
+          const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas)
-          imCvsLocation.top = imageLoc[0].cvsTop
+          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height)
-          imCvsLocation.left = imageLoc[0].cvsLeft
-          imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft
-          imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop
-          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height)
           f7.utils.nextFrame(() => {
             this.setData()
           })
@@ -624,7 +611,12 @@
           if (li >= numBoxes) li -= numBoxes
           return li
         }
-        let boxCoords = this.box2cvs(this.showResults)
+        let boxCoords = []
+        this.resultData.detections.forEach(d => {
+          if (d.aboveThreshold && d.isSearched && !d.isDeleted) {
+            boxCoords.push(d.box.getBoxes('point',this.imageView,this.$refs.image_cvs)[1])
+          }
+        })
         const numBoxes = boxCoords.length
         let clickX = (e.offsetX - this.canvasOffset.x) / this.canvasZoom
         let clickY = (e.offsetY - this.canvasOffset.y) / this.canvasZoom
@@ -633,41 +625,13 @@
         var findBox = boxCoords.findIndex( (r, i) => { 
           let di = loopIndex(i)
           if (di == this.selectedChip ) return false
-          return r.cvsLeft <= clickX &&
+          return r.left <= clickX &&
-            r.cvsRight >= clickX &&
+            r.right >= clickX &&
-            r.cvsTop <= clickY &&
+            r.top <= clickY &&
-            r.cvsBottom >= clickY &&
+            r.bottom >= clickY
-            this.resultData.detections[di].aboveThreshold &&
-            this.resultData.detections[di].isSearched &&
-            !this.resultData.detections[di].isDeleted
         })
         this.selectChip(findBox >= 0 ? this.resultData.detections[loopIndex(findBox)].resultIndex : this.selectedChip)
       },
-      box2cvs(boxInput) {
-        if (!boxInput || boxInput.length == 0) return []
-        const boxList = boxInput.length ? boxInput : [boxInput]
-        const imCanvas = this.$refs.image_cvs
-        var imgWidth
-        var imgHeight
-        const imgAspect = this.imageView.width / this.imageView.height
-        const rendAspect = imCanvas.width / imCanvas.height
-        if (imgAspect >= rendAspect) {
-          imgWidth = imCanvas.width
-          imgHeight = imCanvas.width / imgAspect
-        } else {
-          imgWidth = imCanvas.height * imgAspect
-          imgHeight = imCanvas.height
-        }
-        const cvsCoords = boxList.map( (d, i) => {
-          return {
-            "cvsLeft": (imCanvas.width - imgWidth) / 2 + d.left * imgWidth,
-            "cvsRight": (imCanvas.width - imgWidth) / 2 + d.right * imgWidth,
-            "cvsTop": (imCanvas.height - imgHeight) / 2 + d.top * imgHeight,
-            "cvsBottom": (imCanvas.height - imgHeight) / 2 + d.bottom * imgHeight
-          }
-        })
-        return cvsCoords
-      },
       toggleSettings() {
         this.showDetectSettings = !this.showDetectSettings
         f7.utils.nextFrame(() => {
@@ -707,11 +671,11 @@
       },
       zoomToSelected() {
         const imCanvas = this.$refs.image_cvs
-        const boxCoords = this.box2cvs(this.resultData.detections[this.selectedChip])[0]
+        const boxCoords = this.resultData.detections[this.selectedChip].box.getBoxes('point', this.imageView, imCanvas)
-        const boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
+        const boxWidth = boxCoords[1].right - boxCoords[1].left
-        const boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
+        const boxHeight = boxCoords[1].bottom - boxCoords[1].top
-        const boxMidX = (boxCoords.cvsRight + boxCoords.cvsLeft ) / 2
+        const boxMidX = (boxCoords[1].right + boxCoords[1].left ) / 2
-        const boxMidY = (boxCoords.cvsBottom + boxCoords.cvsTop ) / 2
+        const boxMidY = (boxCoords[1].bottom + boxCoords[1].top ) / 2
         const zoomFactor = Math.min(imCanvas.width / boxWidth * .9, imCanvas.height / boxHeight * .9, 8)
         this.canvasZoom = zoomFactor
         this.canvasOffset.x = -(boxMidX * zoomFactor) + imCanvas.width / 2