From 874901086d916c74358a96dcf0fbf524dd8165db Mon Sep 17 00:00:00 2001 From: Justin Georgi Date: Sat, 12 Oct 2024 02:28:22 +0000 Subject: [PATCH] Add Structure Class (#203) This PR adds a class for detected structures and two additional classes to handle the coordinates of the bounding boxes for those structures. The classes do a better job handling conversions between image pixels, canvas pixels, and screen pixels. This means that they take the place of several chunks of the previous code such as the box2cvs function. Reviewed-on: https://gitea.azgeorgis.net/ALVINN/ALVINN_f7/pulls/203 --- src/js/structures.js | 157 +++++++++++++++++++++++++++++++++++++++++++ src/pages/detect.vue | 124 ++++++++++++---------------------- 2 files changed, 201 insertions(+), 80 deletions(-) create mode 100644 src/js/structures.js diff --git a/src/js/structures.js b/src/js/structures.js new file mode 100644 index 0000000..ffe6a98 --- /dev/null +++ b/src/js/structures.js @@ -0,0 +1,157 @@ +class Coordinate { + constructor(x, y) { + this.x = x + this.y = y + } + + toRefFrame(...frameArgs) { + if (frameArgs.length == 0) { + return {x: this.x, y: this.y} + } + let outFrames = [] + //Get Coordinates in Image Reference Frame + if (frameArgs[0].tagName == 'IMG' && frameArgs[0].width && frameArgs[0].height) { + outFrames.push({ + x: this.x * frameArgs[0].width, + y: this.y * frameArgs[0].height + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Image') + } + //Get Coordinates in Canvas Reference Frame + if (frameArgs[1]) { + if (frameArgs[1].tagName == 'CANVAS' && frameArgs[1].width && frameArgs[1].height) { + let imgWidth + let imgHeight + const imgAspect = frameArgs[0].width / frameArgs[0].height + const rendAspect = frameArgs[1].width / frameArgs[1].height + if (imgAspect >= rendAspect) { + imgWidth = frameArgs[1].width + imgHeight = frameArgs[1].width / imgAspect + } else { + imgWidth = frameArgs[1].height * imgAspect + imgHeight = frameArgs[1].height + } + outFrames.push({ + x: (frameArgs[1].width - imgWidth) / 2 + this.x * imgWidth, + y: (frameArgs[1].height - imgHeight) / 2 + this.y * imgHeight + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Canvas') + } + } + //Get Coordinates in Screen Reference Frame + if (frameArgs[2]) { + if (frameArgs[2].zoom && frameArgs[2].offset && frameArgs[2].offset.x !== undefined && frameArgs[2].offset.y !== undefined) { + outFrames.push({ + x: outFrames[1].x * frameArgs[2].zoom + frameArgs[2].offset.x, + y: outFrames[1].y * frameArgs[2].zoom + frameArgs[2].offset.y + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Screen') + } + } + + return outFrames + } + + toString() { + return `(x: ${this.x}, y: ${this.y})` + } +} + +export class StructureBox { + constructor(top, left, bottom, right) { + this.topLeft = new Coordinate(left, top) + this.bottomRight = new Coordinate(right, bottom) + } + + getBoxes(boxType, ...frameArgs) { + let lowerH, lowerV, calcSide + switch (boxType) { + case 'point': + lowerH = 'right' + lowerV = 'bottom' + break + case 'side': + lowerH = 'width' + lowerV = 'height' + calcSide = true + break + default: + throw new Error(`StructureBox: invalid boxType - ${boxType}`) + } + if (frameArgs.length == 0) { + return { + left: this.topLeft.x, + top: this.topLeft.y, + [lowerH]: this.bottomRight.x - ((calcSide) ? this.topLeft.x : 0), + [lowerV]: this.bottomRight.y - ((calcSide) ? this.topLeft.y : 0) + } + } + const tL = this.topLeft.toRefFrame(...frameArgs) + const bR = this.bottomRight.toRefFrame(...frameArgs) + let outBoxes = [] + tL.forEach((cd, i) => { + outBoxes.push({ + left: cd.x, + top: cd.y, + [lowerH]: bR[i].x - ((calcSide) ? cd.x : 0), + [lowerV]: bR[i].y - ((calcSide) ? cd.y : 0) + }) + }) + return outBoxes + } +} + +export class Structure { + constructor(structResult) { + this.label = structResult.label + this.confidence = structResult.confidence + this.box = new StructureBox( + structResult.top, + structResult.left, + structResult.bottom, + structResult.right + ) + this.deleted = false + this.index = -1 + this.passThreshold = true + this.searched = false + } + + get resultIndex() { + return this.index + } + + set resultIndex(newIdx) { + this.index = newIdx + } + + get isDeleted() { + return this.deleted + } + + set isDeleted(del) { + this.deleted = !!del + } + + get isSearched() { + return this.searched + } + + set isSearched(ser) { + this.searched = !!ser + } + + get aboveThreshold() { + return this.passThreshold + } + + setThreshold(level) { + if (typeof level != 'number') { + throw new Error(`Structure: invalid threshold level ${level}`) + } + this.passThreshold = this.confidence >= level + } +} diff --git a/src/pages/detect.vue b/src/pages/detect.vue index 8939b02..5a52b91 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -144,11 +144,11 @@ import touchMixin from './touch-mixin' import detectionWorker from '@/assets/detect-worker.js?worker&inline' + import { Structure, StructureBox } from '../js/structures' const regions = ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck'] let activeRegion = 4 let classesList = [] - let imCvsLocation = {} let imageLoadMode = "environment" let serverSettings = {} let otherSettings = {} @@ -158,6 +158,7 @@ let detectWorker = null let vidWorker = null let canvasMoving = false + let imageLocation = new StructureBox(0, 0, 1, 1) export default { mixins: [submitMixin, detectionMixin, cameraMixin, touchMixin], @@ -286,14 +287,14 @@ var filteredResults = this.resultData.detections if (!filteredResults) return [] - var allSelect = this.detectorLabels.every( s => { return s.detect } ) - var selectedLabels = this.detectorLabels + const allSelect = this.detectorLabels.every( s => { return s.detect } ) + const selectedLabels = this.detectorLabels .filter( l => { return l.detect }) .map( l => { return l.name }) filteredResults.forEach( (d, i) => { - filteredResults[i].resultIndex = i - filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel - filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label) + d.resultIndex = i + d.setThreshold(this.detectorLevel) + d.isSearched = allSelect || selectedLabels.includes(d.label) }) if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) { @@ -333,15 +334,17 @@ self = this if (eDetect.data.error) { self.detecting = false - self.this.resultData = {} + self.resultData = {} loadFailure() f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`) } else if (eDetect.data.success == 'detection') { self.detecting = false - self.resultData = eDetect.data.detections - if (self.resultData) { - self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name}) - } + self.resultData = {detections: []} + eDetect.data.detections.detections.forEach((d) => { + d.label = self.detectorLabels[d.label].name + let detectedStructure = new Structure(d) + self.resultData.detections.push(detectedStructure) + }) self.uploadDirty = true } else if (eDetect.data.success == 'model') { reloadModel = false @@ -476,33 +479,25 @@ iChip = this.selectedChip } const [imCanvas, imageCtx] = this.resetView(true) - let structLeft = this.imageView.width * this.resultData.detections[iChip].left - let structTop = this.imageView.height * this.resultData.detections[iChip].top - let structWidth = this.imageView.width * (this.resultData.detections[iChip].right - this.resultData.detections[iChip].left) - let structHeight = this.imageView.height * (this.resultData.detections[iChip].bottom - this.resultData.detections[iChip].top) + let structBox, cvsBox, screenBox + [structBox, cvsBox, screenBox] = this.resultData.detections[iChip].box.getBoxes('side', this.imageView, imCanvas, {zoom: this.canvasZoom, offset: {...this.canvasOffset}}) - const boxCoords = this.box2cvs(this.resultData.detections[iChip])[0] + this.infoLinkPos.x = Math.min(Math.max(screenBox.left, 0),imCanvas.width) + this.infoLinkPos.y = Math.min(Math.max(screenBox.top, 0), imCanvas.height) - let boxLeft = boxCoords.cvsLeft - let boxTop = boxCoords.cvsTop - let boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft - let boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop - this.infoLinkPos.x = Math.min(Math.max(boxCoords.cvsLeft * this.canvasZoom + this.canvasOffset.x, 0),imCanvas.width) - this.infoLinkPos.y = Math.min(Math.max(boxCoords.cvsTop * this.canvasZoom + this.canvasOffset.y, 0), imCanvas.height) - - let imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height) - imageCtx.drawImage(this.imageView, structLeft, structTop, structWidth, structHeight, boxLeft, boxTop, boxWidth, boxHeight) + const imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height) + imageCtx.drawImage(this.imageView, structBox.left, structBox.top, structBox.width, structBox.height, cvsBox.left, cvsBox.top, cvsBox.width, cvsBox.height) imageCtx.save() - imageCtx.arc(boxLeft, boxTop, 14 / this.canvasZoom, 0, 2 * Math.PI) + imageCtx.arc(cvsBox.left, cvsBox.top, 14 / this.canvasZoom, 0, 2 * Math.PI) imageCtx.closePath() imageCtx.clip() imageCtx.drawImage(this.imageView, - structLeft - (14 / this.canvasZoom * imageScale), - structTop - (14 / this.canvasZoom * imageScale), + structBox.left - (14 / this.canvasZoom * imageScale), + structBox.top - (14 / this.canvasZoom * imageScale), (28 / this.canvasZoom * imageScale), (28 / this.canvasZoom * imageScale), - boxLeft - (14 / this.canvasZoom), - boxTop - (14 / this.canvasZoom), + cvsBox.left - (14 / this.canvasZoom), + cvsBox.top - (14 / this.canvasZoom), (28 / this.canvasZoom), (28 / this.canvasZoom)) imageCtx.restore() this.selectedChip = iChip @@ -532,13 +527,9 @@ imageCtx.strokeStyle = 'yellow' imageCtx.lineWidth = 3 / this.canvasZoom if (this.imageLoaded) { - let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1}) - imCvsLocation.top = imageLoc[0].cvsTop - imCvsLocation.left = imageLoc[0].cvsLeft - imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft - imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop + const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas) if (drawChip) {imageCtx.globalAlpha = .5} - imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height) + imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height) if (drawChip) {imageCtx.globalAlpha = 1} } this.structureZoomed = false @@ -592,12 +583,8 @@ imCanvas.width = imCanvas.clientWidth imCanvas.height = imCanvas.clientHeight const imageCtx = imCanvas.getContext("2d") - let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1}) - imCvsLocation.top = imageLoc[0].cvsTop - imCvsLocation.left = imageLoc[0].cvsLeft - imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft - imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop - imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height) + const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas) + imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height) f7.utils.nextFrame(() => { this.setData() }) @@ -624,7 +611,12 @@ if (li >= numBoxes) li -= numBoxes return li } - let boxCoords = this.box2cvs(this.showResults) + let boxCoords = [] + this.resultData.detections.forEach(d => { + if (d.aboveThreshold && d.isSearched && !d.isDeleted) { + boxCoords.push(d.box.getBoxes('point',this.imageView,this.$refs.image_cvs)[1]) + } + }) const numBoxes = boxCoords.length let clickX = (e.offsetX - this.canvasOffset.x) / this.canvasZoom let clickY = (e.offsetY - this.canvasOffset.y) / this.canvasZoom @@ -633,41 +625,13 @@ var findBox = boxCoords.findIndex( (r, i) => { let di = loopIndex(i) if (di == this.selectedChip ) return false - return r.cvsLeft <= clickX && - r.cvsRight >= clickX && - r.cvsTop <= clickY && - r.cvsBottom >= clickY && - this.resultData.detections[di].aboveThreshold && - this.resultData.detections[di].isSearched && - !this.resultData.detections[di].isDeleted + return r.left <= clickX && + r.right >= clickX && + r.top <= clickY && + r.bottom >= clickY }) this.selectChip(findBox >= 0 ? this.resultData.detections[loopIndex(findBox)].resultIndex : this.selectedChip) }, - box2cvs(boxInput) { - if (!boxInput || boxInput.length == 0) return [] - const boxList = boxInput.length ? boxInput : [boxInput] - const imCanvas = this.$refs.image_cvs - var imgWidth - var imgHeight - const imgAspect = this.imageView.width / this.imageView.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - const cvsCoords = boxList.map( (d, i) => { - return { - "cvsLeft": (imCanvas.width - imgWidth) / 2 + d.left * imgWidth, - "cvsRight": (imCanvas.width - imgWidth) / 2 + d.right * imgWidth, - "cvsTop": (imCanvas.height - imgHeight) / 2 + d.top * imgHeight, - "cvsBottom": (imCanvas.height - imgHeight) / 2 + d.bottom * imgHeight - } - }) - return cvsCoords - }, toggleSettings() { this.showDetectSettings = !this.showDetectSettings f7.utils.nextFrame(() => { @@ -707,11 +671,11 @@ }, zoomToSelected() { const imCanvas = this.$refs.image_cvs - const boxCoords = this.box2cvs(this.resultData.detections[this.selectedChip])[0] - const boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft - const boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop - const boxMidX = (boxCoords.cvsRight + boxCoords.cvsLeft ) / 2 - const boxMidY = (boxCoords.cvsBottom + boxCoords.cvsTop ) / 2 + const boxCoords = this.resultData.detections[this.selectedChip].box.getBoxes('point', this.imageView, imCanvas) + const boxWidth = boxCoords[1].right - boxCoords[1].left + const boxHeight = boxCoords[1].bottom - boxCoords[1].top + const boxMidX = (boxCoords[1].right + boxCoords[1].left ) / 2 + const boxMidY = (boxCoords[1].bottom + boxCoords[1].top ) / 2 const zoomFactor = Math.min(imCanvas.width / boxWidth * .9, imCanvas.height / boxHeight * .9, 8) this.canvasZoom = zoomFactor this.canvasOffset.x = -(boxMidX * zoomFactor) + imCanvas.width / 2