diff --git a/src/js/structures.js b/src/js/structures.js new file mode 100644 index 0000000..5ba2b8c --- /dev/null +++ b/src/js/structures.js @@ -0,0 +1,157 @@ +class Coordinate { + constructor(x, y) { + this.x = x + this.y = y + } + + toRefFrame(...frameArgs) { + if (frameArgs.length == 0) { + return {x: this.x, y: this.y} + } + let outFrames = [] + //Get Coordinates in Image Reference Frame + if (frameArgs[0].tagName == 'IMG' && frameArgs[0].width && frameArgs[0].height) { + outFrames.push({ + x: this.x * frameArgs[0].width, + y: this.y * frameArgs[0].height + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Image') + } + //Get Coordinates in Canvas Reference Frame + if (frameArgs[1]) { + if (frameArgs[1].tagName == 'CANVAS' && frameArgs[1].width && frameArgs[1].height) { + let imgWidth + let imgHeight + const imgAspect = frameArgs[0].width / frameArgs[0].height + const rendAspect = frameArgs[1].width / frameArgs[1].height + if (imgAspect >= rendAspect) { + imgWidth = frameArgs[1].width + imgHeight = frameArgs[1].width / imgAspect + } else { + imgWidth = frameArgs[1].height * imgAspect + imgHeight = frameArgs[1].height + } + outFrames.push({ + x: (frameArgs[1].width - imgWidth) / 2 + this.x * imgWidth, + y: (frameArgs[1].height - imgHeight) / 2 + this.y * imgHeight + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Canvas') + } + } + //Get Coordinates in Screen Reference Frame + if (frameArgs[2]) { + if (frameArgs[2].zoom && frameArgs[2].offset) { + outFrames.push({ + x: outFrames[1].x * frameArgs[2].zoom + frameArgs[2].offset.x, + y: outFrames[1].y * frameArgs[2].zoom + frameArgs[2].offset.y + }) + } else { + throw new Error('Coordinate: invalid reference frame for frameType: Screen') + } + } + + return outFrames + } + + toString() { + return `(x: ${this.x}, y: ${this.y})` + } +} + +export class StructureBox { + constructor(top, left, bottom, right) { + this.topLeft = new Coordinate(left, top) + this.bottomRight = new Coordinate(right, bottom) + } + + getBoxes(boxType, ...frameArgs) { + let lowerH, lowerV, calcSide + switch (boxType) { + case 'point': + lowerH = 'right' + lowerV = 'bottom' + break + case 'side': + lowerH = 'width' + lowerV = 'height' + calcSide = true + break + default: + throw new Error(`StructureBox: invalid boxType - ${boxType}`) + } + if (frameArgs.length == 0) { + return { + left: this.topLeft.x, + top: this.topLeft.y, + [lowerH]: this.bottomRight.x - ((calcSide) ? this.topLeft.x : 0), + [lowerV]: this.bottomRight.y - ((calcSide) ? this.topLeft.y : 0) + } + } + const tL = this.topLeft.toRefFrame(...frameArgs) + const bR = this.bottomRight.toRefFrame(...frameArgs) + let outBoxes = [] + tL.forEach((cd, i) => { + outBoxes.push({ + left: cd.x, + top: cd.y, + [lowerH]: bR[i].x - ((calcSide) ? cd.x : 0), + [lowerV]: bR[i].y - ((calcSide) ? cd.y : 0) + }) + }) + return outBoxes + } +} + +export default class Structure { + constructor(structResult) { + this.label = structResult.label + this.confidence = structResult.confidence + this.box = new StructureBox( + structResult.top, + structResult.left, + structResult.bottom, + structResult.right + ) + this.deleted = false + this.index = -1 + this.passThreshold = true + this.searched = false + } + + get resultIndex() { + return this.index + } + + set resultIndex(newIdx) { + this.index = newIdx + } + + get isDeleted() { + return this.deleted + } + + set isDeleted(del) { + this.deleted = !!del + } + + get isSearched() { + return this.searched + } + + set isSearched(ser) { + this.searched = !!ser + } + + get aboveThreshold() { + return this.passThreshold + } + + setThreshold(level) { + if (typeof level != 'number') { + throw new Error(`Structure: invalid threshold level ${level}`) + } + this.passThreshold = this.confidence >= level + } +} diff --git a/src/pages/detect.vue b/src/pages/detect.vue index 8939b02..e5ab0e4 100644 --- a/src/pages/detect.vue +++ b/src/pages/detect.vue @@ -144,6 +144,8 @@ import touchMixin from './touch-mixin' import detectionWorker from '@/assets/detect-worker.js?worker&inline' + import Structure from '../js/structures' + import { StructureBox } from '../js/structures' const regions = ['Thorax','Abdomen/Pelvis','Limbs','Head and Neck'] let activeRegion = 4 @@ -158,6 +160,7 @@ let detectWorker = null let vidWorker = null let canvasMoving = false + let imageLocation = new StructureBox(0, 0, 1, 1) export default { mixins: [submitMixin, detectionMixin, cameraMixin, touchMixin], @@ -286,14 +289,14 @@ var filteredResults = this.resultData.detections if (!filteredResults) return [] - var allSelect = this.detectorLabels.every( s => { return s.detect } ) - var selectedLabels = this.detectorLabels + const allSelect = this.detectorLabels.every( s => { return s.detect } ) + const selectedLabels = this.detectorLabels .filter( l => { return l.detect }) .map( l => { return l.name }) filteredResults.forEach( (d, i) => { - filteredResults[i].resultIndex = i - filteredResults[i].aboveThreshold = d.confidence >= this.detectorLevel - filteredResults[i].isSearched = allSelect || selectedLabels.includes(d.label) + d.resultIndex = i + d.setThreshold(this.detectorLevel) + d.isSearched = allSelect || selectedLabels.includes(d.label) }) if (!filteredResults.some( s => s.resultIndex == this.selectedChip && s.aboveThreshold && s.isSearched && !s.isDeleted)) { @@ -333,15 +336,17 @@ self = this if (eDetect.data.error) { self.detecting = false - self.this.resultData = {} + self.resultData = {} loadFailure() f7.dialog.alert(`ALVINN structure finding error: ${eDetect.data.message}`) } else if (eDetect.data.success == 'detection') { self.detecting = false - self.resultData = eDetect.data.detections - if (self.resultData) { - self.resultData.detections.map(d => {d.label = self.detectorLabels[d.label].name}) - } + self.resultData = {detections: []} + eDetect.data.detections.detections.forEach((d) => { + d.label = self.detectorLabels[d.label].name + let detectedStructure = new Structure(d) + self.resultData.detections.push(detectedStructure) + }) self.uploadDirty = true } else if (eDetect.data.success == 'model') { reloadModel = false @@ -476,33 +481,25 @@ iChip = this.selectedChip } const [imCanvas, imageCtx] = this.resetView(true) - let structLeft = this.imageView.width * this.resultData.detections[iChip].left - let structTop = this.imageView.height * this.resultData.detections[iChip].top - let structWidth = this.imageView.width * (this.resultData.detections[iChip].right - this.resultData.detections[iChip].left) - let structHeight = this.imageView.height * (this.resultData.detections[iChip].bottom - this.resultData.detections[iChip].top) + let structBox, cvsBox, screenBox + [structBox, cvsBox, screenBox] = this.resultData.detections[iChip].box.getBoxes('side', this.imageView, imCanvas, {zoom: this.canvasZoom, offset: {...this.canvasOffset}}) - const boxCoords = this.box2cvs(this.resultData.detections[iChip])[0] + this.infoLinkPos.x = Math.min(Math.max(screenBox.left, 0),imCanvas.width) + this.infoLinkPos.y = Math.min(Math.max(screenBox.top, 0), imCanvas.height) - let boxLeft = boxCoords.cvsLeft - let boxTop = boxCoords.cvsTop - let boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft - let boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop - this.infoLinkPos.x = Math.min(Math.max(boxCoords.cvsLeft * this.canvasZoom + this.canvasOffset.x, 0),imCanvas.width) - this.infoLinkPos.y = Math.min(Math.max(boxCoords.cvsTop * this.canvasZoom + this.canvasOffset.y, 0), imCanvas.height) - - let imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height) - imageCtx.drawImage(this.imageView, structLeft, structTop, structWidth, structHeight, boxLeft, boxTop, boxWidth, boxHeight) + const imageScale = Math.max(this.imageView.width / imCanvas.width, this.imageView.height / imCanvas.height) + imageCtx.drawImage(this.imageView, structBox.left, structBox.top, structBox.width, structBox.height, cvsBox.left, cvsBox.top, cvsBox.width, cvsBox.height) imageCtx.save() - imageCtx.arc(boxLeft, boxTop, 14 / this.canvasZoom, 0, 2 * Math.PI) + imageCtx.arc(cvsBox.left, cvsBox.top, 14 / this.canvasZoom, 0, 2 * Math.PI) imageCtx.closePath() imageCtx.clip() imageCtx.drawImage(this.imageView, - structLeft - (14 / this.canvasZoom * imageScale), - structTop - (14 / this.canvasZoom * imageScale), + structBox.left - (14 / this.canvasZoom * imageScale), + structBox.top - (14 / this.canvasZoom * imageScale), (28 / this.canvasZoom * imageScale), (28 / this.canvasZoom * imageScale), - boxLeft - (14 / this.canvasZoom), - boxTop - (14 / this.canvasZoom), + cvsBox.left - (14 / this.canvasZoom), + cvsBox.top - (14 / this.canvasZoom), (28 / this.canvasZoom), (28 / this.canvasZoom)) imageCtx.restore() this.selectedChip = iChip @@ -532,13 +529,9 @@ imageCtx.strokeStyle = 'yellow' imageCtx.lineWidth = 3 / this.canvasZoom if (this.imageLoaded) { - let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1}) - imCvsLocation.top = imageLoc[0].cvsTop - imCvsLocation.left = imageLoc[0].cvsLeft - imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft - imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop + const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas) if (drawChip) {imageCtx.globalAlpha = .5} - imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height) + imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height) if (drawChip) {imageCtx.globalAlpha = 1} } this.structureZoomed = false @@ -592,12 +585,8 @@ imCanvas.width = imCanvas.clientWidth imCanvas.height = imCanvas.clientHeight const imageCtx = imCanvas.getContext("2d") - let imageLoc = this.box2cvs({top: 0,left: 0,right: 1,bottom: 1}) - imCvsLocation.top = imageLoc[0].cvsTop - imCvsLocation.left = imageLoc[0].cvsLeft - imCvsLocation.width = imageLoc[0].cvsRight - imageLoc[0].cvsLeft - imCvsLocation.height = imageLoc[0].cvsBottom - imageLoc[0].cvsTop - imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imCvsLocation.left, imCvsLocation.top, imCvsLocation.width, imCvsLocation.height) + const imageLoc = imageLocation.getBoxes('side', this.imageView, imCanvas) + imageCtx.drawImage(this.imageView, 0, 0, this.imageView.width, this.imageView.height, imageLoc[1].left, imageLoc[1].top, imageLoc[1].width, imageLoc[1].height) f7.utils.nextFrame(() => { this.setData() }) @@ -624,7 +613,12 @@ if (li >= numBoxes) li -= numBoxes return li } - let boxCoords = this.box2cvs(this.showResults) + let boxCoords = [] + this.resultData.detections.forEach(d => { + if (d.aboveThreshold && d.isSearched && !d.isDeleted) { + boxCoords.push(d.box.getBoxes('point',this.imageView,this.$refs.image_cvs)[1]) + } + }) const numBoxes = boxCoords.length let clickX = (e.offsetX - this.canvasOffset.x) / this.canvasZoom let clickY = (e.offsetY - this.canvasOffset.y) / this.canvasZoom @@ -633,41 +627,13 @@ var findBox = boxCoords.findIndex( (r, i) => { let di = loopIndex(i) if (di == this.selectedChip ) return false - return r.cvsLeft <= clickX && - r.cvsRight >= clickX && - r.cvsTop <= clickY && - r.cvsBottom >= clickY && - this.resultData.detections[di].aboveThreshold && - this.resultData.detections[di].isSearched && - !this.resultData.detections[di].isDeleted + return r.left <= clickX && + r.right >= clickX && + r.top <= clickY && + r.bottom >= clickY }) this.selectChip(findBox >= 0 ? this.resultData.detections[loopIndex(findBox)].resultIndex : this.selectedChip) }, - box2cvs(boxInput) { - if (!boxInput || boxInput.length == 0) return [] - const boxList = boxInput.length ? boxInput : [boxInput] - const imCanvas = this.$refs.image_cvs - var imgWidth - var imgHeight - const imgAspect = this.imageView.width / this.imageView.height - const rendAspect = imCanvas.width / imCanvas.height - if (imgAspect >= rendAspect) { - imgWidth = imCanvas.width - imgHeight = imCanvas.width / imgAspect - } else { - imgWidth = imCanvas.height * imgAspect - imgHeight = imCanvas.height - } - const cvsCoords = boxList.map( (d, i) => { - return { - "cvsLeft": (imCanvas.width - imgWidth) / 2 + d.left * imgWidth, - "cvsRight": (imCanvas.width - imgWidth) / 2 + d.right * imgWidth, - "cvsTop": (imCanvas.height - imgHeight) / 2 + d.top * imgHeight, - "cvsBottom": (imCanvas.height - imgHeight) / 2 + d.bottom * imgHeight - } - }) - return cvsCoords - }, toggleSettings() { this.showDetectSettings = !this.showDetectSettings f7.utils.nextFrame(() => { @@ -707,11 +673,11 @@ }, zoomToSelected() { const imCanvas = this.$refs.image_cvs - const boxCoords = this.box2cvs(this.resultData.detections[this.selectedChip])[0] - const boxWidth = boxCoords.cvsRight - boxCoords.cvsLeft - const boxHeight = boxCoords.cvsBottom - boxCoords.cvsTop - const boxMidX = (boxCoords.cvsRight + boxCoords.cvsLeft ) / 2 - const boxMidY = (boxCoords.cvsBottom + boxCoords.cvsTop ) / 2 + const boxCoords = this.resultData.detections[this.selectedChip].box.getBoxes('point', this.imageView, imCanvas) + const boxWidth = boxCoords[1].right - boxCoords[1].left + const boxHeight = boxCoords[1].bottom - boxCoords[1].top + const boxMidX = (boxCoords[1].right + boxCoords[1].left ) / 2 + const boxMidY = (boxCoords[1].bottom + boxCoords[1].top ) / 2 const zoomFactor = Math.min(imCanvas.width / boxWidth * .9, imCanvas.height / boxHeight * .9, 8) this.canvasZoom = zoomFactor this.canvasOffset.x = -(boxMidX * zoomFactor) + imCanvas.width / 2