Use canvas for image rendering (#197)

Closes #194, Closes #189 Instead of making the detection image the background of the canvas element the image is now drawn as part of the canvas. This enables pan and zoom of the image as well. Reviewed-on: #197
2024-09-25 15:05:00 +00:00
parent 8ba930ed2e
commit ab6af04e5b
3 changed files with 179 additions and 38 deletions
--- a/src/components/svg-icon.vue
+++ b/src/components/svg-icon.vue
@@ -17,6 +17,8 @@
    <path v-else-if="icon == 'head'" d="M194-80v-395h80v315h280v-193l105-105q29-29 45-65t16-77q0-40-16.5-76T659-741l-25-26-127 127H347l-43 43-57-56 67-67h160l160-160 82 82q40 40 62 90.5T800-600q0 57-22 107.5T716-402l-82 82v240H194Zm197-187L183-475q-11-11-17-26t-6-31q0-16 6-30.5t17-25.5l84-85 124 123q28 28 43.5 64.5T450-409q0 40-15 76.5T391-267Z"/>
    <path v-else-if="icon == 'photo_sample'" d="M240-80q-33 0-56.5-23.5T160-160v-640q0-33 23.5-56.5T240-880h480q33 0 56.5 23.5T800-800v640q0 33-23.5 56.5T720-80H240Zm0-80h480v-640h-80v280l-100-60-100 60v-280H240v640Zm40-80h400L545-420 440-280l-65-87-95 127Zm-40 80v-640 640Zm200-360 100-60 100 60-100-60-100 60Z"/>
    <path v-else-if="icon == 'reset_slide'" d="M520-330v-60h160v60H520Zm60 210v-50h-60v-60h60v-50h60v160h-60Zm100-50v-60h160v60H680Zm40-110v-160h60v50h60v60h-60v50h-60Zm111-280h-83q-26-88-99-144t-169-56q-117 0-198.5 81.5T200-480q0 72 32.5 132t87.5 98v-110h80v240H160v-80h94q-62-50-98-122.5T120-480q0-75 28.5-140.5t77-114q48.5-48.5 114-77T480-840q129 0 226.5 79.5T831-560Z"/>
+    <path v-else-if="icon == 'zoom_to'" d="M440-40v-167l-44 43-56-56 140-140 140 140-56 56-44-43v167h-80ZM220-340l-56-56 43-44H40v-80h167l-43-44 56-56 140 140-140 140Zm520 0L600-480l140-140 56 56-43 44h167v80H753l43 44-56 56Zm-260-80q-25 0-42.5-17.5T420-480q0-25 17.5-42.5T480-540q25 0 42.5 17.5T540-480q0 25-17.5 42.5T480-420Zm0-180L340-740l56-56 44 43v-167h80v167l44-43 56 56-140 140Z"/>
+    <path v-else-if="icon == 'reset_zoom'" d="M480-320v-100q0-25 17.5-42.5T540-480h100v60H540v100h-60Zm60 240q-25 0-42.5-17.5T480-140v-100h60v100h100v60H540Zm280-240v-100H720v-60h100q25 0 42.5 17.5T880-420v100h-60ZM720-80v-60h100v-100h60v100q0 25-17.5 42.5T820-80H720Zm111-480h-83q-26-88-99-144t-169-56q-117 0-198.5 81.5T200-480q0 72 32.5 132t87.5 98v-110h80v240H160v-80h94q-62-50-98-122.5T120-480q0-75 28.5-140.5t77-114q48.5-48.5 114-77T480-840q129 0 226.5 79.5T831-560Z"/>
  </svg>
 </template>
  
@@ -44,7 +46,9 @@
            'limbs',
            'head',
            'photo_sample',
-            'reset_slide'
+            'reset_slide',
+            'zoom_to',
+            'reset_zoom'
          ]
          return iconList.includes(value)
        }
--- a/src/pages/detect.vue
+++ b/src/pages/detect.vue
@@ -9,15 +9,28 @@
      </f7-nav-right>
    </f7-navbar>
    <f7-block class="detect-grid">
+      <!--<div style="position: absolute;">{{ debugInfo ? JSON.stringify(debugInfo) : "No Info Available" }}</div>-->
      <div class="image-container" ref="image_container">
        <SvgIcon v-if="!imageView.src && !videoAvailable" :icon="f7route.params.region" fill-color="var(--avn-theme-color)"/>
        <div class="vid-container" :style="`display: ${videoAvailable ? 'block' : 'none'}; position: absolute; width: 100%; height: 100%;`">
          <video id="vid-view" ref="vid_viewer" :srcObject="cameraStream" :autoPlay="true" style="width: 100%; height: 100%"></video>
          <f7-button @click="captureVidFrame()" style="position: absolute; bottom: 32px; left: 50%; transform: translateX(-50%); z-index: 3;" fill large>Capture</f7-button>
        </div>
-        <canvas id="im-draw" ref="image_cvs" @click="structureClick" :style="`display: ${(imageLoaded || videoAvailable) ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat; z-index: 2;`" />
+        <canvas
+          id="im-draw"
+          ref="image_cvs"
+          @wheel="spinWheel($event)"
+          @mousedown.middle="startMove($event)"
+          @mousemove="makeMove($event)"
+          @mouseup.middle="endMove($event)"
+          @touchstart="startTouch($event)"
+          @touchend="endTouch($event)"
+          @touchmove="moveTouch($event)"
+          @click="structureClick"
+          :style="`display: ${(imageLoaded || videoAvailable) ? 'block' : 'none'}; flex: 1 1 0%; max-width: 100%; max-height: 100%; min-width: 0; min-height: 0; background-size: contain; background-position: center; background-repeat: no-repeat; z-index: 2;`"
+        ></canvas>
        <f7-link v-if="getInfoUrl && (selectedChip > -1)"
-          :style="`left: ${infoLinkPos.x}px; top: ${infoLinkPos.y}px; transform: translate(calc(-50% - ${infoLinkPos.adj}px),calc(-50% - ${infoLinkPos.adj}px));`"
+          :style="`left: ${infoLinkPos.x}px; top: ${infoLinkPos.y}px; transform: translate(-50%,-50%);`"
          class="structure-info"
          :icon-only="true"
          icon-f7="info"
@@ -61,16 +74,19 @@
        </f7-button>
      </div>
      <f7-segmented class="image-menu" raised>
-        <f7-button popover-open="#region-popover">
-          <RegionIcon :region="activeRegion" :iconSet="getIconSet" />
-        </f7-button>
        <f7-button v-if="!videoAvailable" :class="(!modelLoading) ? '' : 'disabled'" popover-open="#capture-popover">
          <SvgIcon icon="camera_add"/>
        </f7-button>
        <f7-button v-if="videoAvailable" @click="closeCamera()">
          <SvgIcon icon="no_photography"/>
        </f7-button>
-        <f7-button @click="() => showDetectSettings = !showDetectSettings" :class="(imageLoaded) ? '' : 'disabled'">
+        <f7-button v-if="!structureZoomed && selectedChip >= 0" style="height: auto; width: auto;" popover-close="#image-popover" @click="zoomToSelected()">
+          <SvgIcon icon="zoom_to" />
+        </f7-button>
+        <f7-button v-else :class="(canvasZoom != 1) ? '' : 'disabled'" style="height: auto; width: auto;" popover-close="#image-popover" @click="resetZoom()">
+          <SvgIcon icon="reset_zoom" />
+        </f7-button>
+        <f7-button @click="toggleSettings()" :class="(imageLoaded) ? '' : 'disabled'">
          <SvgIcon icon="visibility"/>
          <f7-badge v-if="numResults && (showResults.length != numResults)" color="red" style="position: absolute; right: 15%; top: 15%;">{{ showResults.length - numResults }}</f7-badge>
        </f7-button>
@@ -93,23 +109,6 @@
      </f7-page>
    </f7-panel>

-    <f7-popover id="region-popover" class="popover-button-menu">
-      <f7-segmented raised class="segment-button-menu">
-        <f7-button :class="(getRegions.includes('thorax')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/thorax/" popover-close="#region-popover">
-          <RegionIcon :region="0" :iconSet="getIconSet" />
-        </f7-button>
-        <f7-button :class="(getRegions.includes('abdomen')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/abdomen/" popover-close="#region-popover">
-          <RegionIcon :region="1" :iconSet="getIconSet" />
-        </f7-button>
-        <f7-button :class="(getRegions.includes('limbs')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/limbs/" popover-close="#region-popover">
-          <RegionIcon :region="2" :iconSet="getIconSet" />
-        </f7-button>
-        <f7-button :class="(getRegions.includes('head')) ? '' : ' disabled'" style="height: auto; width: auto;" href="/detect/head/" popover-close="#region-popover">
-          <RegionIcon :region="3" :iconSet="getIconSet" />
-        </f7-button>
-      </f7-segmented>
-    </f7-popover>
-
    <f7-popover id="capture-popover" class="popover-button-menu">
      <f7-segmented raised class="segment-button-menu">
        <f7-button style="height: auto; width: auto;" popover-close="#capture-popover" @click="selectImage('camera')">
@@ -139,11 +138,12 @@
  import submitMixin from './submit-mixin'
  import detectionMixin from './detection-mixin'
  import cameraMixin from './camera-mixin'
+  import touchMixin from './touch-mixin'

  import detectionWorker from '@/assets/detect-worker.js?worker&inline'

  export default {
-    mixins: [submitMixin, detectionMixin, cameraMixin],
+    mixins: [submitMixin, detectionMixin, cameraMixin, touchMixin],
    props: {
      f7route: Object,
    },
@@ -160,6 +160,7 @@
        classesList: [],
        imageLoaded: false,
        imageView: new Image(),
+        imCvsLocation: {},
        imageLoadMode: "environment",
        detecting: false,
        detectPanel: false,
@@ -181,7 +182,12 @@
        cameraStream: null,
        infoLinkPos: {},
        detectWorker: null,
-        vidWorker: null
+        vidWorker: null,
+        canvasMoving: false,
+        canvasOffset: {x: 0, y: 0},
+        canvasZoom: 1,
+        structureZoomed: false,
+        debugInfo: null
      }
    },
    setup() {
@@ -334,6 +340,9 @@
              self.reloadModel = false
              loadSuccess()
            }
+            f7.utils.nextFrame(() => {
+              this.selectChip("redraw")
+            })
          }
        }

@@ -375,6 +384,9 @@
            f7.dialog.alert(`ALVINN structure finding error: ${e.message}`)
          })
        }
+        f7.utils.nextFrame(() => {
+          this.selectChip("redraw")
+        })
      },
      selectAll (ev) {
        if (ev.target.checked) {
@@ -446,10 +458,8 @@
        let boxTop = boxCoords.cvsTop
        let boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
        let boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
-        this.infoLinkPos.x = boxCoords.cvsLeft
-        this.infoLinkPos.y = boxCoords.cvsTop
-        let boxMin = Math.min(boxHeight, boxWidth)
-        this.infoLinkPos.adj = (boxMin >= 50) ? 0 : Math.min(10, 50 - boxMin)
+        this.infoLinkPos.x = Math.min(Math.max(boxCoords.cvsLeft * this.canvasZoom + this.canvasOffset.x, 0),imCanvas.width)
+        this.infoLinkPos.y = Math.min(Math.max(boxCoords.cvsTop * this.canvasZoom + this.canvasOffset.y, 0), imCanvas.height)

        imageCtx.strokeRect(boxLeft, boxTop, boxWidth, boxHeight)
        this.selectedChip = iChip
@@ -473,9 +483,20 @@
        imCanvas.width = imCanvas.clientWidth
        imCanvas.height = imCanvas.clientHeight
        imageCtx.clearRect(0,0,imCanvas.width,imCanvas.height)
+        imageCtx.translate(this.canvasOffset.x,this.canvasOffset.y)
+        imageCtx.scale(this.canvasZoom,this.canvasZoom)
        imageCtx.globalAlpha = 1
        imageCtx.strokeStyle = 'yellow'
-        imageCtx.lineWidth = 3
+        imageCtx.lineWidth = 3 / this.canvasZoom
+        if (this.imageLoaded) {
+          let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1})
+          this.imCvsLocation.top = imageLoc[0].cvsTop
+          this.imCvsLocation.left = imageLoc[0].cvsLeft
+          this.imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft
+          this.imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop
+          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, this.imCvsLocation.left, this.imCvsLocation.top, this.imCvsLocation.width, this.imCvsLocation.height)
+        }
+        this.structureZoomed = false
        return [imCanvas, imageCtx]
      },
      getImage (searchImage) {
@@ -516,8 +537,18 @@
          this.imageView.src = imgData
          return(this.imageView.decode())
        }).then( () => {
-          const [imCanvas, _]  = this.resetView()
-          imCanvas.style['background-image'] = `url(${this.imageView.src})`
+          this.canvasOffset = {x: 0, y: 0}
+          this.canvasZoom = 1
+          const imCanvas = this.$refs.image_cvs
+          imCanvas.width = imCanvas.clientWidth
+          imCanvas.height = imCanvas.clientHeight
+          const imageCtx = imCanvas.getContext("2d")
+          let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1})
+          this.imCvsLocation.top = imageLoc[0].cvsTop
+          this.imCvsLocation.left = imageLoc[0].cvsLeft
+          this.imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft
+          this.imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop
+          imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, this.imCvsLocation.left, this.imCvsLocation.top, this.imCvsLocation.width, this.imCvsLocation.height)
          f7.utils.nextFrame(() => {
            this.setData()
          })
@@ -538,10 +569,12 @@
      },
      structureClick(e) {
        const boxCoords = this.box2cvs(this.showResults)
-        var findBox = boxCoords.findIndex( (r, i) => { return r.cvsLeft <= e.offsetX &&
-            r.cvsRight >= e.offsetX &&
-            r.cvsTop <= e.offsetY &&
-            r.cvsBottom >= e.offsetY &&
+        let clickX = (e.offsetX - this.canvasOffset.x) / this.canvasZoom
+        let clickY = (e.offsetY - this.canvasOffset.y) / this.canvasZoom
+        var findBox = boxCoords.findIndex( (r, i) => { return r.cvsLeft <= clickX &&
+            r.cvsRight >= clickX &&
+            r.cvsTop <= clickY &&
+            r.cvsBottom >= clickY &&
            this.resultData.detections[i].resultIndex > this.selectedChip &&
            this.resultData.detections[i].aboveThreshold &&
            this.resultData.detections[i].isSearched &&
@@ -552,7 +585,7 @@
      box2cvs(boxInput) {
        if (!boxInput || boxInput.length == 0) return []
        const boxList = boxInput.length ? boxInput : [boxInput]
-        const [imCanvas, imageCtx] = this.resetView()
+        const imCanvas = this.$refs.image_cvs
        var imgWidth
        var imgHeight
        const imgAspect = this.imageView.width / this.imageView.height
@@ -573,6 +606,57 @@
          }
        })
        return cvsCoords
+      },
+      toggleSettings() {
+        this.showDetectSettings = !this.showDetectSettings
+        f7.utils.nextFrame(() => {
+          this.selectChip("redraw")
+        })
+      },
+      startMove() {
+        this.canvasMoving = true
+      },
+      endMove() {
+        this.canvasMoving = false
+      },
+      makeMove(event) {
+        if (this.canvasMoving) {
+          this.canvasOffset.x += event.movementX
+          this.canvasOffset.y += event.movementY
+          this.selectChip("redraw")
+        }
+      },
+      spinWheel(event) {
+        let zoomFactor
+        if (event.wheelDelta > 0) {
+          zoomFactor = 1.05
+        } else if (event.wheelDelta < 0) {
+          zoomFactor = 1 / 1.05
+        }
+        this.canvasZoom *= zoomFactor
+        this.canvasOffset.x = event.offsetX * (1 - zoomFactor) + this.canvasOffset.x * zoomFactor
+        this.canvasOffset.y = event.offsetY * (1 - zoomFactor) + this.canvasOffset.y * zoomFactor
+        this.selectChip("redraw")
+      },
+      resetZoom() {
+        this.canvasZoom = 1
+        this.canvasOffset.x = 0
+        this.canvasOffset.y = 0
+        this.selectChip("redraw")
+      },
+      zoomToSelected() {
+        const imCanvas = this.$refs.image_cvs
+        const boxCoords = this.box2cvs(this.resultData.detections[this.selectedChip])[0]
+        const boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft
+        const boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop
+        const boxMidX = (boxCoords.cvsRight + boxCoords.cvsLeft ) / 2
+        const boxMidY = (boxCoords.cvsBottom + boxCoords.cvsTop ) / 2
+        const zoomFactor = Math.min(imCanvas.width / boxWidth * .9, imCanvas.height / boxHeight * .9, 8)
+        this.canvasZoom = zoomFactor
+        this.canvasOffset.x = -(boxMidX * zoomFactor) + imCanvas.width / 2
+        this.canvasOffset.y = -(boxMidY * zoomFactor) + imCanvas.height / 2
+        this.selectChip("redraw")
+        this.structureZoomed = true
      }
    }
  }
--- a/src/pages/touch-mixin.js
+++ b/src/pages/touch-mixin.js
@@ -0,0 +1,53 @@
+export default {
+  data () {
+    return {
+      touchPrevious: {}
+    }
+  },
+  methods: {
+    startTouch(event) {
+      if (event.touches.length == 1) {
+        this.touchPrevious = {x: event.touches[0].clientX, y: event.touches[0].clientY}
+      }
+      if (event.touches.length == 2) {
+        let midX = (event.touches.item(0).clientX + event.touches.item(1).clientX) / 2
+        let midY = (event.touches.item(0).clientY + event.touches.item(1).clientY) / 2
+        this.touchPrevious = {distance: this.touchDistance(event.touches), x: midX, y: midY}
+      }
+    },
+    endTouch(event) {
+      if (event.touches.length == 1) {
+        this.touchPrevious = {x: event.touches[0].clientX, y: event.touches[0].clientY}
+      } else {
+        //this.debugInfo = null
+      }
+    },
+    moveTouch(event) {
+      switch (event.touches.length) {
+        case 1:
+          console.log(event)
+          this.canvasOffset.x += event.touches[0].clientX - this.touchPrevious.x
+          this.canvasOffset.y += event.touches[0].clientY - this.touchPrevious.y
+          this.touchPrevious = {x: event.touches[0].clientX, y: event.touches[0].clientY}
+          break;
+        case 2:
+          let newDistance = this.touchDistance(event.touches)
+          let midX = (event.touches.item(0).clientX + event.touches.item(1).clientX) / 2
+          let midY = (event.touches.item(0).clientY + event.touches.item(1).clientY) / 2
+          let scale = newDistance / this.touchPrevious.distance
+          let scaleChange = this.canvasZoom * (scale - 1)
+          this.canvasZoom *= scale
+          this.canvasOffset.x += -((midX - 16) * scaleChange) + (midX - this.touchPrevious.x)
+          this.canvasOffset.y += -((midY - 96) * scaleChange) + (midY - this.touchPrevious.y)
+          this.touchPrevious = {distance: newDistance, x: midX, y: midY}
+          break;
+      }
+      this.selectChip("redraw")
+    },
+    touchDistance(touches) {
+      let touch1 = touches.item(0)
+      let touch2 = touches.item(1)
+      return Math.sqrt((touch1.clientX - touch2.clientX) ** 2 + (touch1.clientY - touch2.clientY) ** 2)
+    }
+  }
+}