From 0ea4897b1c47e42cc292aad099d8e9628433ae02 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 28 Jul 2021 15:09:06 +0300 Subject: [PATCH] Semi-automatic tools enhancements (Client-side points minimizer) (#3450) * First stage for points minimizer * Fixed issue with correct opencv initialization status * Displaying points during interaction * Added releasing memory * Initial version for on-the-fly optimization * Redesigned accuracy * Updated version & changelog * Fixed opencv scissors * Clean up some intermediate state * Fixed scss * Redesigned slider a bit * Added errored shape * Keep slider hidden while didn't recieve first points * Adjusted settings slider * Updated label * A couple of fixes for trackers & detectors * Updated default value --- CHANGELOG.md | 1 + cvat-canvas/src/scss/canvas.scss | 9 +- .../src/typescript/interactionHandler.ts | 47 ++++++++ cvat-ui/src/actions/settings-actions.ts | 10 ++ .../approximation-accuracy.tsx | 76 +++++++++++++ .../controls-side-bar/opencv-control.tsx | 105 +++++++++++++----- .../controls-side-bar/tools-control.tsx | 95 ++++++++++++++-- .../objects-side-bar/styles.scss | 26 +++++ .../header/settings-modal/styles.scss | 8 +- .../settings-modal/workspace-settings.tsx | 32 +++++- .../settings-modal/workspace-settings.tsx | 8 ++ cvat-ui/src/reducers/interfaces.ts | 1 + cvat-ui/src/reducers/settings-reducer.ts | 10 ++ .../opencv-wrapper/intelligent-scissors.ts | 7 +- .../utils/opencv-wrapper/opencv-wrapper.ts | 37 ++++++ 15 files changed, 423 insertions(+), 49 deletions(-) create mode 100644 cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e66af34f..29bd43929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a tutorial for semi-automatic/automatic annotation () - Explicit "Done" button when drawing any polyshapes () - Histogram equalization with OpenCV javascript () +- Client-side polyshapes approximation when using semi-automatic interactors & scissors () ### Changed diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss index ff0a50fc2..993745ba4 100644 --- a/cvat-canvas/src/scss/canvas.scss +++ b/cvat-canvas/src/scss/canvas.scss @@ -161,11 +161,16 @@ polyline.cvat_canvas_shape_splitting { .cvat_canvas_removable_interaction_point { cursor: - url('data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHZpZXdCb3g9IjAgMCAxMCAxMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTEgMUw5IDlNMSA5TDkgMSIgc3Ryb2tlPSJibGFjayIvPgo8L3N2Zz4K') - 10 10, + url( + 'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHZpZXdCb3g9IjAgMCAxMCAxMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTEgMUw5IDlNMSA5TDkgMSIgc3Ryb2tlPSJibGFjayIvPgo8L3N2Zz4K' + ) 10 10, auto; } +.cvat_canvas_interact_intermediate_shape_point { + pointer-events: none; +} + .svg_select_boundingRect { opacity: 0; pointer-events: none; diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index c6526a4c3..dcb8101ef 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -225,6 +225,7 @@ export class InteractionHandlerImpl implements InteractionHandler { private release(): void { if (this.drawnIntermediateShape) { + this.selectize(false, this.drawnIntermediateShape); this.drawnIntermediateShape.remove(); this.drawnIntermediateShape = null; } @@ -270,21 +271,25 @@ export class InteractionHandlerImpl implements InteractionHandler { private updateIntermediateShape(): void { const { intermediateShape, geometry } = this; if (this.drawnIntermediateShape) { + this.selectize(false, this.drawnIntermediateShape); this.drawnIntermediateShape.remove(); } if (!intermediateShape) return; const { shapeType, points } = intermediateShape; if (shapeType === 'polygon') { + const erroredShape = shapeType === 'polygon' && points.length < 3 * 2; this.drawnIntermediateShape = this.canvas .polygon(stringifyPoints(translateToCanvas(geometry.offset, points))) .attr({ 'color-rendering': 'optimizeQuality', 'shape-rendering': 'geometricprecision', 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + stroke: erroredShape ? 'red' : 'black', fill: 'none', }) .addClass('cvat_canvas_interact_intermediate_shape'); + this.selectize(true, this.drawnIntermediateShape, erroredShape); } else { throw new Error( `Shape type "${shapeType}" was not implemented at interactionHandler::updateIntermediateShape`, @@ -292,6 +297,39 @@ export class InteractionHandlerImpl implements InteractionHandler { } } + private selectize(value: boolean, shape: SVG.Element, erroredShape = false): void { + const self = this; + + if (value) { + (shape as any).selectize(value, { + deepSelect: true, + pointSize: consts.BASE_POINT_SIZE / self.geometry.scale, + rotationPoint: false, + classPoints: 'cvat_canvas_interact_intermediate_shape_point', + pointType(cx: number, cy: number): SVG.Circle { + return this.nested + .circle(this.options.pointSize) + .stroke(erroredShape ? 'red' : 'black') + .fill('black') + .center(cx, cy) + .attr({ + 'fill-opacity': 1, + 'stroke-width': consts.POINTS_STROKE_WIDTH / self.geometry.scale, + }); + }, + }); + } else { + (shape as any).selectize(false, { + deepSelect: true, + }); + } + + const handler = shape.remember('_selectHandler'); + if (handler && handler.nested) { + handler.nested.fill(shape.attr('fill')); + } + } + public constructor( onInteraction: ( shapes: InteractionResult[] | null, @@ -398,6 +436,15 @@ export class InteractionHandlerImpl implements InteractionHandler { shape.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); } } + + for (const element of window.document.getElementsByClassName('cvat_canvas_interact_intermediate_shape_point')) { + element.setAttribute('stroke-width', `${consts.POINTS_STROKE_WIDTH / (2 * this.geometry.scale)}`); + element.setAttribute('r', `${consts.BASE_POINT_SIZE / this.geometry.scale}`); + } + + if (this.drawnIntermediateShape) { + this.drawnIntermediateShape.stroke({ width: consts.BASE_STROKE_WIDTH / this.geometry.scale }); + } } public interact(interactionData: InteractionData): void { diff --git a/cvat-ui/src/actions/settings-actions.ts b/cvat-ui/src/actions/settings-actions.ts index 6fca99c3d..966743c47 100644 --- a/cvat-ui/src/actions/settings-actions.ts +++ b/cvat-ui/src/actions/settings-actions.ts @@ -26,6 +26,7 @@ export enum SettingsActionTypes { SWITCH_AUTO_SAVE = 'SWITCH_AUTO_SAVE', CHANGE_AUTO_SAVE_INTERVAL = 'CHANGE_AUTO_SAVE_INTERVAL', CHANGE_AAM_ZOOM_MARGIN = 'CHANGE_AAM_ZOOM_MARGIN', + CHANGE_DEFAULT_APPROX_POLY_THRESHOLD = 'CHANGE_DEFAULT_APPROX_POLY_THRESHOLD', SWITCH_AUTOMATIC_BORDERING = 'SWITCH_AUTOMATIC_BORDERING', SWITCH_INTELLIGENT_POLYGON_CROP = 'SWITCH_INTELLIGENT_POLYGON_CROP', SWITCH_SHOWNIG_INTERPOLATED_TRACKS = 'SWITCH_SHOWNIG_INTERPOLATED_TRACKS', @@ -270,6 +271,15 @@ export function switchSettingsDialog(show?: boolean): AnyAction { }; } +export function changeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): AnyAction { + return { + type: SettingsActionTypes.CHANGE_DEFAULT_APPROX_POLY_THRESHOLD, + payload: { + approxPolyAccuracy, + }, + }; +} + export function setSettings(settings: Partial): AnyAction { return { type: SettingsActionTypes.SET_SETTINGS, diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy.tsx new file mode 100644 index 000000000..2bfbf7fb3 --- /dev/null +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy.tsx @@ -0,0 +1,76 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import React, { CSSProperties } from 'react'; +import ReactDOM from 'react-dom'; +import Text from 'antd/lib/typography/Text'; +import Slider from 'antd/lib/slider'; +import { Col, Row } from 'antd/lib/grid'; + +interface Props { + approxPolyAccuracy: number; + onChange(value: number): void; +} + +export const MAX_ACCURACY = 13; + +export const marks: Record = {}; +marks[0] = { + style: { + color: '#1890ff', + }, + label: less, +}; +marks[MAX_ACCURACY] = { + style: { + color: '#61c200', + }, + label: more, +}; + +export function thresholdFromAccuracy(approxPolyAccuracy: number): number { + const approxPolyMaxDistance = MAX_ACCURACY - approxPolyAccuracy; + let threshold = 0; + if (approxPolyMaxDistance > 0) { + if (approxPolyMaxDistance <= 8) { + // −2.75x+7y+1=0 linear made from two points (1; 0.25) and (8; 3) + threshold = (2.75 * approxPolyMaxDistance - 1) / 7; + } else { + // 4 for 9, 8 for 10, 16 for 11, 32 for 12, 64 for 13 + threshold = 2 ** (approxPolyMaxDistance - 7); + } + } + + return threshold; +} + +function ApproximationAccuracy(props: Props): React.ReactPortal | null { + const { approxPolyAccuracy, onChange } = props; + const target = window.document.getElementsByClassName('cvat-canvas-container')[0]; + + return target ? + ReactDOM.createPortal( + + + Points: + + + + + , + target, + ) : + null; +} + +export default React.memo(ApproximationAccuracy); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx index f2f86766c..2543a3ef5 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx @@ -30,6 +30,9 @@ import { } from 'actions/annotation-actions'; import LabelSelector from 'components/label-selector/label-selector'; import CVATTooltip from 'components/common/cvat-tooltip'; +import ApproximationAccuracy, { + thresholdFromAccuracy, +} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy'; import { ImageProcessing } from 'utils/opencv-wrapper/opencv-interfaces'; import withVisibilityHandling from './handle-popover-visibility'; @@ -41,6 +44,7 @@ interface Props { states: any[]; frame: number; curZOrder: number; + defaultApproxPolyAccuracy: number; frameData: any; } @@ -57,6 +61,7 @@ interface State { initializationError: boolean; initializationProgress: number; activeLabelID: number; + approxPolyAccuracy: number; activeImageModifiers: ImageModifier[]; } @@ -81,11 +86,15 @@ function mapStateToProps(state: CombinedState): Props { frame: { number: frame, data: frameData }, }, }, + settings: { + workspace: { defaultApproxPolyAccuracy }, + }, } = state; return { isActivated: activeControl === ActiveControl.OPENCV_TOOLS, canvasInstance: canvasInstance as Canvas, + defaultApproxPolyAccuracy, jobInstance, curZOrder, labels, @@ -105,18 +114,21 @@ const mapDispatchToProps = { class OpenCVControlComponent extends React.PureComponent { private activeTool: IntelligentScissors | null; + private latestPoints: number[]; private canvasForceUpdateWasEnabled: boolean; public constructor(props: Props & DispatchToProps) { super(props); - const { labels } = props; + const { labels, defaultApproxPolyAccuracy } = props; this.activeTool = null; + this.latestPoints = []; this.canvasForceUpdateWasEnabled = false; this.state = { libraryInitialized: openCVWrapper.isInitialized, initializationError: false, initializationProgress: -1, + approxPolyAccuracy: defaultApproxPolyAccuracy, activeLabelID: labels.length ? labels[0].id : null, activeImageModifiers: [], }; @@ -128,14 +140,35 @@ class OpenCVControlComponent extends React.PureComponent => { + const { approxPolyAccuracy } = this.state; const { createAnnotations, isActivated, jobInstance, frame, labels, curZOrder, canvasInstance, } = this.props; @@ -160,24 +194,32 @@ class OpenCVControlComponent extends React.PureComponent label.id === activeLabelID)[0], - // need to recalculate without the latest sliding point - points: await this.runCVAlgorithm(pressedPoints, threshold), + points: openCVWrapper.contours + .approxPoly(finalPoints, thresholdFromAccuracy(approxPolyAccuracy)) + .flat(), occluded: false, zOrder: curZOrder, }); @@ -253,19 +295,6 @@ class OpenCVControlComponent extends React.PureComponent ) : ( - - - + <> + { + if (libraryInitialized !== openCVWrapper.isInitialized) { + this.setState({ + libraryInitialized: openCVWrapper.isInitialized, + }); + } + }} + > + + + {isActivated ? ( + { + this.setState({ approxPolyAccuracy: value }); + }} + /> + ) : null} + ); } } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index d859c8057..9f4ff760b 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -13,6 +13,7 @@ import Text from 'antd/lib/typography/Text'; import Tabs from 'antd/lib/tabs'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; +import message from 'antd/lib/message'; import Progress from 'antd/lib/progress'; import InputNumber from 'antd/lib/input-number'; @@ -20,6 +21,7 @@ import { AIToolsIcon } from 'icons'; import { Canvas, convertShapesForInteractor } from 'cvat-canvas-wrapper'; import range from 'utils/range'; import getCore from 'cvat-core-wrapper'; +import openCVWrapper from 'utils/opencv-wrapper/opencv-wrapper'; import { CombinedState, ActiveControl, Model, ObjectType, ShapeType, } from 'reducers/interfaces'; @@ -31,6 +33,9 @@ import { } from 'actions/annotation-actions'; import DetectorRunner from 'components/model-runner-modal/detector-runner'; import LabelSelector from 'components/label-selector/label-selector'; +import ApproximationAccuracy, { + thresholdFromAccuracy, +} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy'; import withVisibilityHandling from './handle-popover-visibility'; interface StateToProps { @@ -46,6 +51,7 @@ interface StateToProps { trackers: Model[]; curZOrder: number; aiToolsRef: MutableRefObject; + defaultApproxPolyAccuracy: number; } interface DispatchToProps { @@ -60,6 +66,7 @@ const CustomPopover = withVisibilityHandling(Popover, 'tools-control'); function mapStateToProps(state: CombinedState): StateToProps { const { annotation } = state; + const { settings } = state; const { number: frame } = annotation.player.frame; const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; @@ -79,6 +86,7 @@ function mapStateToProps(state: CombinedState): StateToProps { frame, curZOrder: annotation.annotations.zLayer.cur, aiToolsRef: annotation.aiToolsRef, + defaultApproxPolyAccuracy: settings.workspace.defaultApproxPolyAccuracy, }; } @@ -97,14 +105,16 @@ interface State { trackingProgress: number | null; trackingFrames: number; fetching: boolean; + pointsRecieved: boolean; + approxPolyAccuracy: number; mode: 'detection' | 'interaction' | 'tracking'; } export class ToolsControlComponent extends React.PureComponent { private interactionIsAborted: boolean; - private interactionIsDone: boolean; - private latestResult: number[]; + private latestResponseResult: number[][]; + private latestResult: number[][]; public constructor(props: Props) { super(props); @@ -112,12 +122,15 @@ export class ToolsControlComponent extends React.PureComponent { activeInteractor: props.interactors.length ? props.interactors[0] : null, activeTracker: props.trackers.length ? props.trackers[0] : null, activeLabelID: props.labels.length ? props.labels[0].id : null, + approxPolyAccuracy: props.defaultApproxPolyAccuracy, trackingProgress: null, trackingFrames: 10, fetching: false, + pointsRecieved: false, mode: 'interaction', }; + this.latestResponseResult = []; this.latestResult = []; this.interactionIsAborted = false; this.interactionIsDone = false; @@ -130,16 +143,39 @@ export class ToolsControlComponent extends React.PureComponent { canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); } - public componentDidUpdate(prevProps: Props): void { - const { isActivated } = this.props; + public componentDidUpdate(prevProps: Props, prevState: State): void { + const { isActivated, defaultApproxPolyAccuracy, canvasInstance } = this.props; + const { approxPolyAccuracy, activeInteractor } = this.state; + if (prevProps.isActivated && !isActivated) { window.removeEventListener('contextmenu', this.contextmenuDisabler); } else if (!prevProps.isActivated && isActivated) { // reset flags when start interaction/tracking + this.setState({ + approxPolyAccuracy: defaultApproxPolyAccuracy, + pointsRecieved: false, + }); + this.latestResult = []; + this.latestResponseResult = []; this.interactionIsDone = false; this.interactionIsAborted = false; window.addEventListener('contextmenu', this.contextmenuDisabler); } + + if (prevState.approxPolyAccuracy !== approxPolyAccuracy) { + if (isActivated && activeInteractor !== null && this.latestResponseResult.length) { + this.approximateResponsePoints(this.latestResponseResult).then((points: number[][]) => { + this.latestResult = points; + canvasInstance.interact({ + enabled: true, + intermediateShape: { + shapeType: ShapeType.POLYGON, + points: this.latestResult.flat(), + }, + }); + }); + } + } } public componentWillUnmount(): void { @@ -197,20 +233,26 @@ export class ToolsControlComponent extends React.PureComponent { if ((e as CustomEvent).detail.shapesUpdated) { this.setState({ fetching: true }); try { - this.latestResult = await core.lambda.call(jobInstance.task, interactor, { + this.latestResponseResult = await core.lambda.call(jobInstance.task, interactor, { frame, pos_points: convertShapesForInteractor((e as CustomEvent).detail.shapes, 0), neg_points: convertShapesForInteractor((e as CustomEvent).detail.shapes, 2), }); + this.latestResult = this.latestResponseResult; + if (this.interactionIsAborted) { // while the server request // user has cancelled interaction (for example pressed ESC) + // need to clean variables that have been just set this.latestResult = []; + this.latestResponseResult = []; return; } + + this.latestResult = await this.approximateResponsePoints(this.latestResponseResult); } finally { - this.setState({ fetching: false }); + this.setState({ fetching: false, pointsRecieved: !!this.latestResult.length }); } } @@ -259,7 +301,8 @@ export class ToolsControlComponent extends React.PureComponent { const { activeLabelID } = this.state; const [label] = jobInstance.task.labels.filter((_label: any): boolean => _label.id === activeLabelID); - if (!(e as CustomEvent).detail.isDone) { + const { isDone, shapesUpdated } = (e as CustomEvent).detail; + if (!isDone || !shapesUpdated) { return; } @@ -320,8 +363,27 @@ export class ToolsControlComponent extends React.PureComponent { }); }; + private async approximateResponsePoints(points: number[][]): Promise { + const { approxPolyAccuracy } = this.state; + if (points.length > 3) { + if (!openCVWrapper.isInitialized) { + const hide = message.loading('OpenCV.js initialization..'); + try { + await openCVWrapper.initialize(() => {}); + } finally { + hide(); + } + } + + const threshold = thresholdFromAccuracy(approxPolyAccuracy); + return openCVWrapper.contours.approxPoly(points, threshold); + } + + return points; + } + public async trackState(state: any): Promise { - const { jobInstance, frame } = this.props; + const { jobInstance, frame, fetchAnnotations } = this.props; const { activeTracker, trackingFrames } = this.state; const { clientID, points } = state; @@ -373,6 +435,7 @@ export class ToolsControlComponent extends React.PureComponent { } } finally { this.setState({ trackingProgress: null, fetching: false }); + fetchAnnotations(); } } @@ -577,7 +640,7 @@ export class ToolsControlComponent extends React.PureComponent { private renderDetectorBlock(): JSX.Element { const { - jobInstance, detectors, curZOrder, frame, + jobInstance, detectors, curZOrder, frame, createAnnotations, } = this.props; if (!detectors.length) { @@ -616,7 +679,7 @@ export class ToolsControlComponent extends React.PureComponent { }), ); - createAnnotationsAsync(jobInstance, frame, states); + createAnnotations(jobInstance, frame, states); } catch (error) { notification.error({ description: error.toString(), @@ -661,7 +724,9 @@ export class ToolsControlComponent extends React.PureComponent { const { interactors, detectors, trackers, isActivated, canvasInstance, labels, } = this.props; - const { fetching, trackingProgress } = this.state; + const { + fetching, trackingProgress, approxPolyAccuracy, activeInteractor, pointsRecieved, + } = this.state; if (![...interactors, ...detectors, ...trackers].length) return null; @@ -701,6 +766,14 @@ export class ToolsControlComponent extends React.PureComponent { )} + {isActivated && activeInteractor !== null && pointsRecieved ? ( + { + this.setState({ approxPolyAccuracy: value }); + }} + /> + ) : null} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/styles.scss index 667278fd0..2bf1fa3c9 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/styles.scss @@ -397,3 +397,29 @@ .cvat-objects-sidebar-label-item-disabled { opacity: 0.5; } + +.cvat-workspace-settings-approx-poly-threshold { + .ant-slider-track { + background: linear-gradient(90deg, #1890ff 0%, #61c200 100%); + } +} + +.cvat-approx-poly-threshold-wrapper { + @extend .cvat-workspace-settings-approx-poly-threshold; + + width: $grid-unit-size * 32; + position: absolute; + background: $background-color-2; + top: 8px; + left: 50%; + border-radius: 6px; + border: 1px solid $border-color-3; + z-index: 100; + padding: $grid-unit-size / 2 $grid-unit-size * 2 $grid-unit-size / 2 $grid-unit-size / 2; + + .ant-slider-mark { + position: static; + margin-top: 4px; + pointer-events: none; + } +} diff --git a/cvat-ui/src/components/header/settings-modal/styles.scss b/cvat-ui/src/components/header/settings-modal/styles.scss index 06fd21a9d..a1a0dbc23 100644 --- a/cvat-ui/src/components/header/settings-modal/styles.scss +++ b/cvat-ui/src/components/header/settings-modal/styles.scss @@ -27,7 +27,9 @@ .cvat-workspace-settings-autoborders, .cvat-workspace-settings-intelligent-polygon-cropping, .cvat-workspace-settings-show-text-always, -.cvat-workspace-settings-show-interpolated { +.cvat-workspace-settings-show-interpolated, +.cvat-workspace-settings-approx-poly-threshold, +.cvat-workspace-settings-aam-zoom-margin { margin-bottom: 25px; > div:first-child { @@ -35,6 +37,10 @@ } } +.cvat-workspace-settings-approx-poly-threshold { + user-select: none; +} + .cvat-player-settings-step, .cvat-player-settings-speed, .cvat-player-settings-reset-zoom, diff --git a/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx b/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx index 1aef81dec..48299819d 100644 --- a/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx +++ b/cvat-ui/src/components/header/settings-modal/workspace-settings.tsx @@ -8,7 +8,12 @@ import { Row, Col } from 'antd/lib/grid'; import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox'; import InputNumber from 'antd/lib/input-number'; import Text from 'antd/lib/typography/Text'; +import Slider from 'antd/lib/slider'; +import { + MAX_ACCURACY, + marks, +} from 'components/annotation-page/standard-workspace/controls-side-bar/approximation-accuracy'; import { clamp } from 'utils/math'; interface Props { @@ -19,16 +24,18 @@ interface Props { showObjectsTextAlways: boolean; automaticBordering: boolean; intelligentPolygonCrop: boolean; + defaultApproxPolyAccuracy: number; onSwitchAutoSave(enabled: boolean): void; onChangeAutoSaveInterval(interval: number): void; onChangeAAMZoomMargin(margin: number): void; + onChangeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): void; onSwitchShowingInterpolatedTracks(enabled: boolean): void; onSwitchShowingObjectsTextAlways(enabled: boolean): void; onSwitchAutomaticBordering(enabled: boolean): void; onSwitchIntelligentPolygonCrop(enabled: boolean): void; } -export default function WorkspaceSettingsComponent(props: Props): JSX.Element { +function WorkspaceSettingsComponent(props: Props): JSX.Element { const { autoSave, autoSaveInterval, @@ -37,6 +44,7 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element { showObjectsTextAlways, automaticBordering, intelligentPolygonCrop, + defaultApproxPolyAccuracy, onSwitchAutoSave, onChangeAutoSaveInterval, onChangeAAMZoomMargin, @@ -44,6 +52,7 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element { onSwitchShowingObjectsTextAlways, onSwitchAutomaticBordering, onSwitchIntelligentPolygonCrop, + onChangeDefaultApproxPolyAccuracy, } = props; const minAutoSaveInterval = 1; @@ -168,6 +177,27 @@ export default function WorkspaceSettingsComponent(props: Props): JSX.Element { /> + + + Default number of points in polygon approximation + + + + + + Works for serverless interactors and OpenCV scissors + + ); } + +export default React.memo(WorkspaceSettingsComponent); diff --git a/cvat-ui/src/containers/header/settings-modal/workspace-settings.tsx b/cvat-ui/src/containers/header/settings-modal/workspace-settings.tsx index 2384c1664..d92f44297 100644 --- a/cvat-ui/src/containers/header/settings-modal/workspace-settings.tsx +++ b/cvat-ui/src/containers/header/settings-modal/workspace-settings.tsx @@ -13,6 +13,7 @@ import { switchShowingObjectsTextAlways, switchAutomaticBordering, switchIntelligentPolygonCrop, + changeDefaultApproxPolyAccuracy, } from 'actions/settings-actions'; import { CombinedState } from 'reducers/interfaces'; @@ -25,6 +26,7 @@ interface StateToProps { aamZoomMargin: number; showAllInterpolationTracks: boolean; showObjectsTextAlways: boolean; + defaultApproxPolyAccuracy: number; automaticBordering: boolean; intelligentPolygonCrop: boolean; } @@ -37,6 +39,7 @@ interface DispatchToProps { onSwitchShowingObjectsTextAlways(enabled: boolean): void; onSwitchAutomaticBordering(enabled: boolean): void; onSwitchIntelligentPolygonCrop(enabled: boolean): void; + onChangeDefaultApproxPolyAccuracy(approxPolyAccuracy: number): void; } function mapStateToProps(state: CombinedState): StateToProps { @@ -49,6 +52,7 @@ function mapStateToProps(state: CombinedState): StateToProps { showObjectsTextAlways, automaticBordering, intelligentPolygonCrop, + defaultApproxPolyAccuracy, } = workspace; return { @@ -59,6 +63,7 @@ function mapStateToProps(state: CombinedState): StateToProps { showObjectsTextAlways, automaticBordering, intelligentPolygonCrop, + defaultApproxPolyAccuracy, }; } @@ -85,6 +90,9 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { onSwitchIntelligentPolygonCrop(enabled: boolean): void { dispatch(switchIntelligentPolygonCrop(enabled)); }, + onChangeDefaultApproxPolyAccuracy(threshold: number): void { + dispatch(changeDefaultApproxPolyAccuracy(threshold)); + }, }; } diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index eaae78a48..7da382b85 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -556,6 +556,7 @@ export interface WorkspaceSettingsState { showObjectsTextAlways: boolean; showAllInterpolationTracks: boolean; intelligentPolygonCrop: boolean; + defaultApproxPolyAccuracy: number; } export interface ShapesSettingsState { diff --git a/cvat-ui/src/reducers/settings-reducer.ts b/cvat-ui/src/reducers/settings-reducer.ts index 8e5a66e55..301ff9a7e 100644 --- a/cvat-ui/src/reducers/settings-reducer.ts +++ b/cvat-ui/src/reducers/settings-reducer.ts @@ -31,6 +31,7 @@ const defaultState: SettingsState = { showObjectsTextAlways: false, showAllInterpolationTracks: false, intelligentPolygonCrop: true, + defaultApproxPolyAccuracy: 9, }, player: { canvasBackgroundColor: '#ffffff', @@ -277,6 +278,15 @@ export default (state = defaultState, action: AnyAction): SettingsState => { }, }; } + case SettingsActionTypes.CHANGE_DEFAULT_APPROX_POLY_THRESHOLD: { + return { + ...state, + workspace: { + ...state.workspace, + defaultApproxPolyAccuracy: action.payload.approxPolyAccuracy, + }, + }; + } case SettingsActionTypes.SWITCH_SETTINGS_DIALOG: { return { ...state, diff --git a/cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts b/cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts index cd05eec50..8fca23014 100644 --- a/cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts +++ b/cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts @@ -92,7 +92,6 @@ export default class IntelligentScissorsImplementation implements IntelligentSci if (points.length > 1) { let matImage = null; const contour = new cv.Mat(); - const approx = new cv.Mat(); try { const [prev, cur] = points.slice(-2); @@ -123,11 +122,10 @@ export default class IntelligentScissorsImplementation implements IntelligentSci tool.applyImage(matImage); tool.buildMap(new cv.Point(prevX, prevY)); tool.getContour(new cv.Point(curX, curY), contour); - cv.approxPolyDP(contour, approx, 2, false); const pathSegment = []; - for (let row = 0; row < approx.rows; row++) { - pathSegment.push(approx.intAt(row, 0) + offsetX, approx.intAt(row, 1) + offsetY); + for (let row = 0; row < contour.rows; row++) { + pathSegment.push(contour.intAt(row, 0) + offsetX, contour.intAt(row, 1) + offsetY); } state.anchors[points.length - 1] = { point: cur, @@ -140,7 +138,6 @@ export default class IntelligentScissorsImplementation implements IntelligentSci } contour.delete(); - approx.delete(); } } else { state.path.push(...pointsToNumberArray(applyOffset(points.slice(-1), -offsetX, -offsetY))); diff --git a/cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts b/cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts index d337e1e60..cc18cecbd 100644 --- a/cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts +++ b/cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts @@ -15,6 +15,10 @@ export interface Segmentation { intelligentScissorsFactory: () => IntelligentScissors; } +export interface Contours { + approxPoly: (points: number[] | any, threshold: number, closed?: boolean) => number[][]; +} + export interface ImgProc { hist: () => HistogramEqualization } @@ -85,6 +89,39 @@ export class OpenCVWrapper { return this.initialized; } + public get contours(): Contours { + if (!this.initialized) { + throw new Error('Need to initialize OpenCV first'); + } + + const { cv } = this; + return { + approxPoly: (points: number[] | number[][], threshold: number, closed = true): number[][] => { + const isArrayOfArrays = Array.isArray(points[0]); + if (points.length < 3) { + // one pair of coordinates [x, y], approximation not possible + return (isArrayOfArrays ? points : [points]) as number[][]; + } + const rows = isArrayOfArrays ? points.length : points.length / 2; + const cols = 2; + + const approx = new cv.Mat(); + const contour = cv.matFromArray(rows, cols, cv.CV_32FC1, points.flat()); + try { + cv.approxPolyDP(contour, approx, threshold, closed); // approx output type is CV_32F + const result = []; + for (let row = 0; row < approx.rows; row++) { + result.push([approx.floatAt(row, 0), approx.floatAt(row, 1)]); + } + return result; + } finally { + approx.delete(); + contour.delete(); + } + }, + }; + } + public get segmentation(): Segmentation { if (!this.initialized) { throw new Error('Need to initialize OpenCV first'); -- GitLab