Intelligent scissors with OpenCV javascript (#2689)

* Some UI implementations

* Added opencv wrapper

* Updated Opencv wrapper

* Moved initialization stub

* Added threshold

* Setup interaction with canvas

* Fixed couple of issues

* Added threshold, changing size via ctrl

* tmp

* Aborted host change

* Fixed threshold

* Aborted host

* Some fixes

* Using ready label selector

* Raw implementation

* Added additional arguments

* Fixed some minor issues

* Removed unused file

* Fixed tool reset

* Added short instructions to update opencv.js

* Fixed corner case

* Added error handler, opencv version, updated cvat_proxy & mod_wsgi

* OpenCV returned back

* Using dinamic function instead of script

* Updated changelog & version

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - CVAT-3D: support lidar data on the server side (<https://github.com/openvinotoolkit/cvat/pull/2534>)
+- Intelligent scissors with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/2689>)
 
 ### Changed
 

cvat-canvas/package-lock.json

 {
   "name": "cvat-canvas",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-canvas/package.json

 {
   "name": "cvat-canvas",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

cvat-canvas/src/scss/canvas.scss

@@ -51,6 +51,10 @@ polyline.cvat_shape_drawing_opacity {
     stroke: red;
 }
 
+.cvat_canvas_threshold {
+    stroke: red;
+}
+
 .cvat_canvas_shape_grouping {
     @extend .cvat_shape_action_dasharray;
     @extend .cvat_shape_action_opacity;

cvat-canvas/src/typescript/canvasModel.ts

-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -76,6 +76,10 @@ export interface InteractionData {
     crosshair?: boolean;
     minPosVertices?: number;
     minNegVertices?: number;
+    enableNegVertices?: boolean;
+    enableThreshold?: boolean;
+    enableSliding?: boolean;
+    allowRemoveOnlyLast?: boolean;
 }
 
 export interface InteractionResult {

cvat-canvas/src/typescript/canvasView.ts

@@ -165,6 +165,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
         shapes: InteractionResult[] | null,
         shapesUpdated: boolean = true,
         isDone: boolean = false,
+        threshold: number | null = null,
     ): void {
         const { zLayer } = this.controller;
         if (Array.isArray(shapes)) {
@@ -176,6 +177,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
                     isDone,
                     shapes,
                     zOrder: zLayer || 0,
+                    threshold,
                 },
             });
 
@@ -1050,6 +1052,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
         });
 
         this.content.addEventListener('wheel', (event): void => {
+            if (event.ctrlKey) return;
             const { offset } = this.controller.geometry;
             const point = translateToSVG(this.content, [event.clientX, event.clientY]);
             self.controller.zoom(point[0] - offset, point[1] - offset, event.deltaY > 0 ? -1 : 1);

cvat-canvas/src/typescript/interactionHandler.ts

-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -24,6 +24,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
     private interactionShapes: SVG.Shape[];
     private currentInteractionShape: SVG.Shape | null;
     private crosshair: Crosshair;
+    private threshold: SVG.Rect | null;
+    private thresholdRectSize: number;
 
     private prepareResult(): InteractionResult[] {
         return this.interactionShapes.map(
@@ -63,12 +65,21 @@ export class InteractionHandlerImpl implements InteractionHandler {
             return enabled && !ctrlKey && !!interactionShapes.length;
         }
 
-        const minimumVerticesAchieved =
-            (typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length) &&
-            (typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length);
+        const minPosVerticesAchieved = typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length;
+        const minNegVerticesAchieved = typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length;
+        const minimumVerticesAchieved = minPosVerticesAchieved && minNegVerticesAchieved;
         return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated;
     }
 
+    private addThreshold(): void {
+        const { x, y } = this.cursorPosition;
+        this.threshold = this.canvas
+            .rect(this.thresholdRectSize, this.thresholdRectSize)
+            .fill('none')
+            .addClass('cvat_canvas_threshold');
+        this.threshold.center(x, y);
+    }
+
     private addCrosshair(): void {
         const { x, y } = this.cursorPosition;
         this.crosshair.show(this.canvas, x, y, this.geometry.scale);
@@ -80,9 +91,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
     private interactPoints(): void {
         const eventListener = (e: MouseEvent): void => {
-            if ((e.button === 0 || e.button === 2) && !e.altKey) {
+            if ((e.button === 0 || (e.button === 2 && this.interactionData.enableNegVertices)) && !e.altKey) {
                 e.preventDefault();
                 const [cx, cy] = translateToSVG((this.canvas.node as any) as SVGSVGElement, [e.clientX, e.clientY]);
+                if (!this.isWithingFrame(cx, cy)) return;
+                if (!this.isWithinThreshold(cx, cy)) return;
+
                 this.currentInteractionShape = this.canvas
                     .circle((consts.BASE_POINT_SIZE * 2) / this.geometry.scale)
                     .center(cx, cy)
@@ -101,6 +115,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
                 const self = this.currentInteractionShape;
                 self.on('mouseenter', (): void => {
+                    if (this.interactionData.allowRemoveOnlyLast) {
+                        if (this.interactionShapes.indexOf(self) !== this.interactionShapes.length - 1) {
+                            return;
+                        }
+                    }
+
                     self.attr({
                         'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale,
                     });
@@ -166,6 +186,10 @@ export class InteractionHandlerImpl implements InteractionHandler {
         if (this.interactionData.crosshair) {
             this.addCrosshair();
         }
+
+        if (this.interactionData.enableThreshold) {
+            this.addThreshold();
+        }
     }
 
     private startInteraction(): void {
@@ -183,6 +207,11 @@ export class InteractionHandlerImpl implements InteractionHandler {
             this.removeCrosshair();
         }
 
+        if (this.threshold) {
+            this.threshold.remove();
+            this.threshold = null;
+        }
+
         this.canvas.off('mousedown.interaction');
         this.interactionShapes.forEach((shape: SVG.Shape): SVG.Shape => shape.remove());
         this.interactionShapes = [];
@@ -192,14 +221,39 @@ export class InteractionHandlerImpl implements InteractionHandler {
         }
     }
 
+    private isWithinThreshold(x: number, y: number): boolean {
+        const [prev] = this.interactionShapes.slice(-1);
+        if (!this.interactionData.enableThreshold || !prev) {
+            return true;
+        }
+
+        const [prevCx, prevCy] = [(prev as SVG.Circle).cx(), (prev as SVG.Circle).cy()];
+        const xDiff = Math.abs(prevCx - x);
+        const yDiff = Math.abs(prevCy - y);
+
+        return xDiff < this.thresholdRectSize / 2 && yDiff < this.thresholdRectSize / 2;
+    }
+
+    private isWithingFrame(x: number, y: number): boolean {
+        const { offset, image } = this.geometry;
+        const { width, height } = image;
+        const [imageX, imageY] = [Math.round(x - offset), Math.round(y - offset)];
+        return imageX >= 0 && imageX < width && imageY >= 0 && imageY < height;
+    }
+
     public constructor(
-        onInteraction: (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean) => void,
+        onInteraction: (
+            shapes: InteractionResult[] | null,
+            shapesUpdated?: boolean,
+            isDone?: boolean,
+            threshold?: number,
+        ) => void,
         canvas: SVG.Container,
         geometry: Geometry,
     ) {
         this.onInteraction = (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean): void => {
             this.shapesWereUpdated = false;
-            onInteraction(shapes, shapesUpdated, isDone);
+            onInteraction(shapes, shapesUpdated, isDone, this.threshold ? this.thresholdRectSize / 2 : null);
         };
         this.canvas = canvas;
         this.geometry = geometry;
@@ -208,6 +262,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
         this.interactionData = { enabled: false };
         this.currentInteractionShape = null;
         this.crosshair = new Crosshair();
+        this.threshold = null;
+        this.thresholdRectSize = 300;
         this.cursorPosition = {
             x: 0,
             y: 0,
@@ -219,6 +275,43 @@ export class InteractionHandlerImpl implements InteractionHandler {
             if (this.crosshair) {
                 this.crosshair.move(x, y);
             }
+            if (this.threshold) {
+                this.threshold.center(x, y);
+            }
+
+            if (this.interactionData.enableSliding && this.interactionShapes.length) {
+                if (this.isWithingFrame(x, y)) {
+                    if (this.interactionData.enableThreshold && !this.isWithinThreshold(x, y)) return;
+                    this.onInteraction(
+                        [
+                            ...this.prepareResult(),
+                            {
+                                points: [x - this.geometry.offset, y - this.geometry.offset],
+                                shapeType: 'points',
+                                button: 0,
+                            },
+                        ],
+                        true,
+                        false,
+                    );
+                }
+            }
+        });
+
+        this.canvas.on('wheel.interaction', (e: WheelEvent): void => {
+            if (e.ctrlKey) {
+                if (this.threshold) {
+                    const { x, y } = this.cursorPosition;
+                    e.preventDefault();
+                    if (e.deltaY > 0) {
+                        this.thresholdRectSize *= 6 / 5;
+                    } else {
+                        this.thresholdRectSize *= 5 / 6;
+                    }
+                    this.threshold.size(this.thresholdRectSize, this.thresholdRectSize);
+                    this.threshold.center(x, y);
+                }
+            }
         });
 
         document.body.addEventListener('keyup', (e: KeyboardEvent): void => {

cvat-core/src/ml-model.js

-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -17,6 +17,7 @@ class MLModel {
         this._params = {
             canvas: {
                 minPosVertices: data.min_pos_points,
+                enableNegVertices: true,
             },
         };
     }

cvat-ui/package-lock.json

 {
   "name": "cvat-ui",
-  "version": "1.13.8",
+  "version": "1.14.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-ui/package.json

 {
   "name": "cvat-ui",
-  "version": "1.13.8",
+  "version": "1.14.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/annotation-actions.ts

@@ -18,6 +18,7 @@ import {
     ContextMenuType,
     Workspace,
     Model,
+    OpenCVTool,
 } from 'reducers/interfaces';
 
 import getCore from 'cvat-core-wrapper';
@@ -1354,7 +1355,10 @@ export function pasteShapeAsync(): ThunkAction {
     };
 }
 
-export function interactWithCanvas(activeInteractor: Model, activeLabelID: number): AnyAction {
+export function interactWithCanvas(
+    activeInteractor: Model | OpenCVTool,
+    activeLabelID: number,
+): AnyAction {
     return {
         type: AnnotationActionTypes.INTERACT_WITH_CANVAS,
         payload: {

cvat-ui/src/assets/opencv.svg

+<!-- 
+    The file has been downloaded from: https://icon-icons.com/ru/%D0%B7%D0%BD%D0%B0%D1%87%D0%BE%D0%BA/%D0%92-%D1%84%D0%BE%D1%80%D0%BC%D0%B0%D1%82%D0%B5-OpenCV/132129 
+    License:  Attribution 4.0 International (CC BY 4.0) https://creativecommons.org/licenses/by/4.0/
+    The file has been modified
+-->
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="40" height="40">
+<g style="transform: scale(0.078)">
+	<path d="M148.6458282,81.0641403C191.8570709-0.3458452,307.612915-4.617764,356.5062561,73.3931732c37.8880615,60.4514771,13.7960815,135.4847717-41.8233948,167.7876129l-36.121521-62.5643005c22.1270447-12.8510284,31.7114563-42.7013397,16.6385498-66.750618c-19.4511414-31.034935-65.5021057-29.3354645-82.692749,3.0517044c-12.7206879,23.9658356-2.6391449,51.5502472,18.3088379,63.7294922l-36.1482544,62.6105804C142.0118256,210.643219,116.6704254,141.3057709,148.6458282,81.0641403z M167.9667206,374.4708557c-0.0435791,24.2778625-18.934967,46.8978271-46.092804,47.9000549c-36.6418304,1.3522339-61.0877724-37.6520386-43.8971252-70.0392151c13.2918015-25.0418091,43.8297424-31.7192383,65.9928284-19.1222839l36.2165222-62.7288513c-55.7241974-31.7991638-132.6246796-15.0146027-166.0706635,47.9976501c-43.2111893,81.4099731,18.2372913,179.4530945,110.3418884,176.0540161c68.1375427-2.5146179,115.5750122-59.1652527,115.8612366-120.0613708H167.9667206z M451.714386,270.7571411l-36.1215515,62.5642395c22.2027588,12.816864,31.8418274,42.7249451,16.744751,66.8127441c-19.4511414,31.0349426-65.5021057,29.3354797-82.692688-3.0516968c-12.742218-24.0063782-2.6048279-51.643219,18.4154358-63.7908325l-36.1482544-62.6105652c-52.7280579,30.5827942-78.1254272,99.9726562-46.128479,160.2548218c43.2111816,81.4099731,158.9670105,85.6818848,207.8603821,7.6710205C531.5561523,378.1168213,507.4096069,303.0259705,451.714386,270.7571411z"/>
+</g>
+</svg>

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx

@@ -15,6 +15,7 @@ import MoveControl from './move-control';
 import FitControl from './fit-control';
 import ResizeControl from './resize-control';
 import ToolsControl from './tools-control';
+import OpenCVControl from './opencv-control';
 import DrawRectangleControl from './draw-rectangle-control';
 import DrawPolygonControl from './draw-polygon-control';
 import DrawPolylineControl from './draw-polyline-control';
@@ -90,6 +91,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                 ActiveControl.DRAW_RECTANGLE,
                 ActiveControl.DRAW_CUBOID,
                 ActiveControl.AI_TOOLS,
+                ActiveControl.OPENCV_TOOLS,
             ].includes(activeControl);
 
             if (!drawing) {
@@ -103,7 +105,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                     repeatDrawShape();
                 }
             } else {
-                if (activeControl === ActiveControl.AI_TOOLS) {
+                if ([ActiveControl.AI_TOOLS, ActiveControl.OPENCV_TOOLS].includes(activeControl)) {
                     // separated API method
                     canvasInstance.interact({ enabled: false });
                     return;
@@ -187,6 +189,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
 
             <hr />
             <ToolsControl />
+            <OpenCVControl />
             <DrawRectangleControl
                 canvasInstance={canvasInstance}
                 isDrawing={activeControl === ActiveControl.DRAW_RECTANGLE}

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import React from 'react';
+import { connect } from 'react-redux';
+import { Row, Col } from 'antd/lib/grid';
+import Tooltip from 'antd/lib/tooltip';
+import Popover from 'antd/lib/popover';
+import Icon, { ScissorOutlined } from '@ant-design/icons';
+import Text from 'antd/lib/typography/Text';
+import Tabs from 'antd/lib/tabs';
+import Button from 'antd/lib/button';
+import Progress from 'antd/lib/progress';
+import notification from 'antd/lib/notification';
+
+import { OpenCVIcon } from 'icons';
+import { Canvas, convertShapesForInteractor } from 'cvat-canvas-wrapper';
+import getCore from 'cvat-core-wrapper';
+import openCVWrapper from 'utils/opencv-wrapper/opencv-wrapper';
+import { IntelligentScissors } from 'utils/opencv-wrapper/intelligent-scissors';
+import {
+    CombinedState, ActiveControl, OpenCVTool, ObjectType,
+} from 'reducers/interfaces';
+import {
+    interactWithCanvas,
+    fetchAnnotationsAsync,
+    updateAnnotationsAsync,
+    createAnnotationsAsync,
+} from 'actions/annotation-actions';
+import LabelSelector from 'components/label-selector/label-selector';
+
+interface Props {
+    labels: any[];
+    canvasInstance: Canvas;
+    jobInstance: any;
+    isActivated: boolean;
+    states: any[];
+    frame: number;
+    curZOrder: number;
+}
+
+interface DispatchToProps {
+    onInteractionStart(activeInteractor: OpenCVTool, activeLabelID: number): void;
+    updateAnnotations(statesToUpdate: any[]): void;
+    createAnnotations(sessionInstance: any, frame: number, statesToCreate: any[]): void;
+    fetchAnnotations(): void;
+}
+
+interface State {
+    libraryInitialized: boolean;
+    initializationError: boolean;
+    initializationProgress: number;
+    activeLabelID: number;
+}
+
+const core = getCore();
+
+function mapStateToProps(state: CombinedState): Props {
+    const {
+        annotation: {
+            annotations: {
+                states,
+                zLayer: { cur: curZOrder },
+            },
+            job: { instance: jobInstance, labels },
+            canvas: { activeControl, instance: canvasInstance },
+            player: {
+                frame: { number: frame },
+            },
+        },
+    } = state;
+
+    return {
+        isActivated: activeControl === ActiveControl.OPENCV_TOOLS,
+        canvasInstance,
+        jobInstance,
+        curZOrder,
+        labels,
+        states,
+        frame,
+    };
+}
+
+const mapDispatchToProps = {
+    onInteractionStart: interactWithCanvas,
+    updateAnnotations: updateAnnotationsAsync,
+    fetchAnnotations: fetchAnnotationsAsync,
+    createAnnotations: createAnnotationsAsync,
+};
+
+class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps, State> {
+    private activeTool: IntelligentScissors | null;
+    private interactiveStateID: number | null;
+    private interactionIsDone: boolean;
+
+    public constructor(props: Props & DispatchToProps) {
+        super(props);
+        const { labels } = props;
+
+        this.activeTool = null;
+        this.interactiveStateID = null;
+        this.interactionIsDone = false;
+
+        this.state = {
+            libraryInitialized: openCVWrapper.isInitialized,
+            initializationError: false,
+            initializationProgress: -1,
+            activeLabelID: labels[0].id,
+        };
+    }
+
+    public componentDidMount(): void {
+        const { canvasInstance } = this.props;
+        canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener);
+    }
+
+    public componentDidUpdate(prevProps: Props): void {
+        const { isActivated } = this.props;
+        if (!prevProps.isActivated && isActivated) {
+            // reset flags when before using a tool
+            if (this.activeTool) {
+                this.activeTool.reset();
+            }
+            this.interactiveStateID = null;
+            this.interactionIsDone = false;
+        }
+    }
+
+    public componentWillUnmount(): void {
+        const { canvasInstance } = this.props;
+        canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener);
+    }
+
+    private getInteractiveState(): any | null {
+        const { states } = this.props;
+        return states.filter((_state: any): boolean => _state.clientID === this.interactiveStateID)[0] || null;
+    }
+
+    private cancelListener = async (): Promise<void> => {
+        const {
+            fetchAnnotations, isActivated, jobInstance, frame,
+        } = this.props;
+
+        if (isActivated) {
+            if (this.interactiveStateID !== null) {
+                const state = this.getInteractiveState();
+                this.interactiveStateID = null;
+                await state.delete(frame);
+                fetchAnnotations();
+            }
+
+            await jobInstance.actions.freeze(false);
+        }
+    };
+
+    private interactionListener = async (e: Event): Promise<void> => {
+        const {
+            fetchAnnotations, updateAnnotations, isActivated, jobInstance, frame, labels, curZOrder,
+        } = this.props;
+        const { activeLabelID } = this.state;
+        if (!isActivated || !this.activeTool) {
+            return;
+        }
+
+        const {
+            shapesUpdated, isDone, threshold, shapes,
+        } = (e as CustomEvent).detail;
+        const pressedPoints = convertShapesForInteractor(shapes).flat();
+        this.interactionIsDone = isDone;
+
+        try {
+            let points: number[] = [];
+            if (shapesUpdated) {
+                points = await this.runCVAlgorithm(pressedPoints, threshold);
+            }
+
+            if (this.interactiveStateID === null) {
+                if (!this.interactionIsDone) {
+                    await jobInstance.actions.freeze(true);
+                }
+
+                const object = new core.classes.ObjectState({
+                    ...this.activeTool.params.shape,
+                    frame,
+                    objectType: ObjectType.SHAPE,
+                    label: labels.filter((label: any) => label.id === activeLabelID)[0],
+                    points,
+                    occluded: false,
+                    zOrder: curZOrder,
+                });
+                // need a clientID of a created object to interact with it further
+                // so, we do not use createAnnotationAction
+                const [clientID] = await jobInstance.annotations.put([object]);
+                this.interactiveStateID = clientID;
+
+                // update annotations on a canvas
+                fetchAnnotations();
+                return;
+            }
+
+            const state = this.getInteractiveState();
+            if ((e as CustomEvent).detail.isDone) {
+                const finalObject = new core.classes.ObjectState({
+                    frame: state.frame,
+                    objectType: state.objectType,
+                    label: state.label,
+                    shapeType: state.shapeType,
+                    // need to recalculate without the latest sliding point
+                    points: points = await this.runCVAlgorithm(pressedPoints, threshold),
+                    occluded: state.occluded,
+                    zOrder: state.zOrder,
+                });
+                this.interactiveStateID = null;
+                await state.delete(frame);
+                await jobInstance.actions.freeze(false);
+                await jobInstance.annotations.put([finalObject]);
+                fetchAnnotations();
+            } else {
+                state.points = points;
+                updateAnnotations([state]);
+                fetchAnnotations();
+            }
+        } catch (error) {
+            notification.error({
+                description: error.toString(),
+                message: 'Processing error occured',
+            });
+        }
+    };
+
+    private async runCVAlgorithm(pressedPoints: number[], threshold: number): Promise<number[]> {
+        // Getting image data
+        const canvas: HTMLCanvasElement | undefined = window.document.getElementById('cvat_canvas_background') as
+            | HTMLCanvasElement
+            | undefined;
+        if (!canvas) {
+            throw new Error('Element #cvat_canvas_background was not found');
+        }
+
+        const { width, height } = canvas;
+        const context = canvas.getContext('2d');
+        if (!context) {
+            throw new Error('Canvas context is empty');
+        }
+
+        const [x, y] = pressedPoints.slice(-2);
+        const startX = Math.round(Math.max(0, x - threshold));
+        const startY = Math.round(Math.max(0, y - threshold));
+        const segmentWidth = Math.min(2 * threshold, width - startX);
+        const segmentHeight = Math.min(2 * threshold, height - startY);
+        const imageData = context.getImageData(startX, startY, segmentWidth, segmentHeight);
+
+        if (!this.activeTool) return [];
+
+        // Handling via OpenCV.js
+        const points = await this.activeTool.run(pressedPoints, imageData, startX, startY);
+
+        // Increasing number of points artificially
+        let minNumberOfPoints = 1;
+        // eslint-disable-next-line: eslintdot-notation
+        if (this.activeTool.params.shape.shapeType === 'polyline') {
+            minNumberOfPoints = 2;
+        } else if (this.activeTool.params.shape.shapeType === 'polygon') {
+            minNumberOfPoints = 3;
+        }
+        while (points.length < minNumberOfPoints * 2) {
+            points.push(...points.slice(points.length - 2));
+        }
+
+        return points;
+    }
+
+    private renderDrawingContent(): JSX.Element {
+        const { activeLabelID } = this.state;
+        const { labels, canvasInstance, onInteractionStart } = this.props;
+
+        return (
+            <>
+                <Row justify='center'>
+                    <Col span={24}>
+                        <LabelSelector
+                            style={{ width: '100%' }}
+                            labels={labels}
+                            value={activeLabelID}
+                            onChange={(label: any) => this.setState({ activeLabelID: label.id })}
+                        />
+                    </Col>
+                </Row>
+                <Row justify='start' className='cvat-opencv-drawing-tools'>
+                    <Col>
+                        <Tooltip title='Intelligent scissors' className='cvat-opencv-drawing-tool'>
+                            <Button
+                                onClick={() => {
+                                    this.activeTool = openCVWrapper.segmentation.intelligentScissorsFactory();
+                                    canvasInstance.cancel();
+                                    onInteractionStart(this.activeTool, activeLabelID);
+                                    canvasInstance.interact({
+                                        enabled: true,
+                                        ...this.activeTool.params.canvas,
+                                    });
+                                }}
+                            >
+                                <ScissorOutlined />
+                            </Button>
+                        </Tooltip>
+                    </Col>
+                </Row>
+            </>
+        );
+    }
+
+    private renderContent(): JSX.Element {
+        const { libraryInitialized, initializationProgress, initializationError } = this.state;
+
+        return (
+            <div className='cvat-opencv-control-popover-content'>
+                <Row justify='start'>
+                    <Col>
+                        <Text className='cvat-text-color' strong>
+                            OpenCV
+                        </Text>
+                    </Col>
+                </Row>
+                {libraryInitialized ? (
+                    <Tabs tabBarGutter={8}>
+                        <Tabs.TabPane key='drawing' tab='Drawing' className='cvat-opencv-control-tabpane'>
+                            {this.renderDrawingContent()}
+                        </Tabs.TabPane>
+                        <Tabs.TabPane disabled key='image' tab='Image' className='cvat-opencv-control-tabpane' />
+                    </Tabs>
+                ) : (
+                    <>
+                        <Row justify='start' align='middle'>
+                            <Col span={initializationProgress >= 0 ? 17 : 24}>
+                                <Button
+                                    disabled={initializationProgress !== -1}
+                                    className='cvat-opencv-initialization-button'
+                                    onClick={async () => {
+                                        try {
+                                            this.setState({
+                                                initializationError: false,
+                                                initializationProgress: 0,
+                                            });
+                                            await openCVWrapper.initialize((progress: number) => {
+                                                this.setState({ initializationProgress: progress });
+                                            });
+                                            this.setState({ libraryInitialized: true });
+                                        } catch (error) {
+                                            notification.error({
+                                                description: error.toString(),
+                                                message: 'Could not initialize OpenCV library',
+                                            });
+                                            this.setState({
+                                                initializationError: true,
+                                                initializationProgress: -1,
+                                            });
+                                        }
+                                    }}
+                                >
+                                    Load OpenCV
+                                </Button>
+                            </Col>
+                            {initializationProgress >= 0 && (
+                                <Col span={6} offset={1}>
+                                    <Progress
+                                        width={8 * 5}
+                                        percent={initializationProgress}
+                                        type='circle'
+                                        status={initializationError ? 'exception' : undefined}
+                                    />
+                                </Col>
+                            )}
+                        </Row>
+                    </>
+                )}
+            </div>
+        );
+    }
+
+    public render(): JSX.Element {
+        const { isActivated, canvasInstance } = this.props;
+        const dynamcPopoverPros = isActivated ?
+            {
+                overlayStyle: {
+                    display: 'none',
+                },
+            } :
+            {};
+
+        const dynamicIconProps = isActivated ?
+            {
+                className: 'cvat-active-canvas-control cvat-opencv-control',
+                onClick: (): void => {
+                    canvasInstance.interact({ enabled: false });
+                },
+            } :
+            {
+                className: 'cvat-tools-control',
+            };
+
+        return (
+            <Popover
+                {...dynamcPopoverPros}
+                placement='right'
+                overlayClassName='cvat-opencv-control-popover'
+                content={this.renderContent()}
+            >
+                <Icon {...dynamicIconProps} component={OpenCVIcon} />
+            </Popover>
+        );
+    }
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(OpenCVControlComponent);

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx

@@ -17,7 +17,7 @@ import Progress from 'antd/lib/progress';
 import InputNumber from 'antd/lib/input-number';
 
 import { AIToolsIcon } from 'icons';
-import { Canvas } from 'cvat-canvas-wrapper';
+import { Canvas, convertShapesForInteractor } from 'cvat-canvas-wrapper';
 import range from 'utils/range';
 import getCore from 'cvat-core-wrapper';
 import {
@@ -29,7 +29,6 @@ import {
     updateAnnotationsAsync,
     createAnnotationsAsync,
 } from 'actions/annotation-actions';
-import { InteractionResult } from 'cvat-canvas/src/typescript/canvas';
 import DetectorRunner from 'components/model-runner-modal/detector-runner';
 import LabelSelector from 'components/label-selector/label-selector';
 import withVisibilityHandling from './handle-popover-visibility';
@@ -89,22 +88,6 @@ const mapDispatchToProps = {
     createAnnotations: createAnnotationsAsync,
 };
 
-function convertShapesForInteractor(shapes: InteractionResult[]): number[][] {
-    const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => {
-        if (!(index % 2)) {
-            // 0, 2, 4
-            acc.push([array[index], array[index + 1]]);
-        }
-        return acc;
-    };
-
-    return shapes
-        .filter((shape: InteractionResult): boolean => shape.shapeType === 'points' && shape.button === 0)
-        .map((shape: InteractionResult): number[] => shape.points)
-        .flat()
-        .reduce(reducer, []);
-}
-
 type Props = StateToProps & DispatchToProps;
 interface State {
     activeInteractor: Model | null;
@@ -218,11 +201,11 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         } = this.props;
         const { activeInteractor, interactiveStateID, fetching } = this.state;
 
-        try {
-            if (!isActivated) {
-                throw Error('Canvas raises event "canvas.interacted" when interaction with it is off');
-            }
+        if (!isActivated) {
+            return;
+        }
 
+        try {
             if (fetching) {
                 this.interactionIsDone = (e as CustomEvent).detail.isDone;
                 return;
@@ -320,8 +303,17 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
 
     private onTracking = async (e: Event): Promise<void> => {
         const {
-            isActivated, jobInstance, frame, curZOrder, fetchAnnotations,
+            isActivated,
+            jobInstance,
+            frame,
+            curZOrder,
+            fetchAnnotations,
         } = this.props;
+
+        if (!isActivated) {
+            return;
+        }
+
         const { activeLabelID } = this.state;
         const [label] = jobInstance.task.labels.filter((_label: any): boolean => _label.id === activeLabelID);
 
@@ -331,10 +323,6 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
 
         this.interactionIsDone = true;
         try {
-            if (!isActivated) {
-                throw Error('Canvas raises event "canvas.interacted" when interaction with it is off');
-            }
-
             const { points } = (e as CustomEvent).detail.shapes[0];
             const state = new core.classes.ObjectState({
                 shapeType: ShapeType.RECTANGLE,
@@ -550,9 +538,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                             className='cvat-tools-track-button'
                             disabled={!activeTracker || fetching || frame === jobInstance.stopFrame}
                             onClick={() => {
-                                this.setState({
-                                    mode: 'tracking',
-                                });
+                                this.setState({ mode: 'tracking' });
 
                                 if (activeTracker) {
                                     canvasInstance.cancel();
@@ -625,9 +611,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                             className='cvat-tools-interact-button'
                             disabled={!activeInteractor || fetching}
                             onClick={() => {
-                                this.setState({
-                                    mode: 'interaction',
-                                });
+                                this.setState({ mode: 'interaction' });
 
                                 if (activeInteractor) {
                                     canvasInstance.cancel();
@@ -673,30 +657,21 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                 task={jobInstance.task}
                 runInference={async (task: any, model: Model, body: object) => {
                     try {
-                        this.setState({
-                            mode: 'detection',
-                        });
-
-                        this.setState({ fetching: true });
-                        const result = await core.lambda.call(task, model, {
-                            ...body,
-                            frame,
-                        });
-
-                        const states = result.map(
-                            (data: any): any =>
-                                new core.classes.ObjectState({
-                                    shapeType: data.type,
-                                    label: task.labels.filter((label: any): boolean => label.name === data.label)[0],
-                                    points: data.points,
-                                    objectType: ObjectType.SHAPE,
-                                    frame,
-                                    occluded: false,
-                                    source: 'auto',
-                                    attributes: {},
-                                    zOrder: curZOrder,
-                                }),
-                        );
+                        this.setState({ mode: 'detection', fetching: true });
+                        const result = await core.lambda.call(task, model, { ...body, frame });
+                        const states = result.map((data: any): any => (
+                            new core.classes.ObjectState({
+                                shapeType: data.type,
+                                label: task.labels.filter((label: any): boolean => label.name === data.label)[0],
+                                points: data.points,
+                                objectType: ObjectType.SHAPE,
+                                frame,
+                                occluded: false,
+                                source: 'auto',
+                                attributes: {},
+                                zOrder: curZOrder,
+                            })
+                        ));
 
                         await jobInstance.annotations.put(states);
                         fetchAnnotations();

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -55,7 +55,8 @@
 .cvat-group-control,
 .cvat-split-track-control,
 .cvat-issue-control,
-.cvat-tools-control {
+.cvat-tools-control,
+.cvat-opencv-control {
     border-radius: 3.3px;
     transform: scale(0.65);
     padding: 2px;
@@ -100,6 +101,7 @@
 }
 
 .cvat-draw-shape-popover,
+.cvat-opencv-control-popover,
 .cvat-setup-tag-popover,
 .cvat-tools-control-popover {
     .ant-popover-inner-content {
@@ -117,22 +119,47 @@
     width: 100%;
 }
 
-.cvat-tools-control-popover-content {
+.cvat-tools-control-popover-content,
+.cvat-opencv-control-popover-content {
     width: fit-content;
-    padding: 10px;
-    border-radius: 5px;
+    padding: $grid-unit-size;
+    border-radius: $grid-unit-size;
     background: $background-color-2;
+
+    .ant-tabs-tab {
+        width: $grid-unit-size * 14;
+        justify-content: center;
+    }
+}
+
+.cvat-opencv-initialization-button {
+    width: 100%;
+    margin: $grid-unit-size 0;
+}
+
+.cvat-opencv-drawing-tools {
+    margin-top: $grid-unit-size;
+}
+
+.cvat-opencv-drawing-tool {
+    padding: $grid-unit-size;
+    width: $grid-unit-size * 5;
+    height: $grid-unit-size * 5;
+
+    > i {
+        font-size: 16px;
+    }
 }
 
 .cvat-setup-tag-popover-content,
 .cvat-draw-shape-popover-content {
-    padding: 10px;
-    border-radius: 5px;
+    padding: $grid-unit-size;
+    border-radius: $grid-unit-size;
     background: $background-color-2;
-    width: 270px;
+    width: 34 * $grid-unit-size;
 
     > div {
-        margin-top: 5px;
+        margin-top: $grid-unit-size;
     }
 
     > div:nth-child(3) > div > div {

cvat-ui/src/cvat-canvas-wrapper.ts

@@ -12,6 +12,22 @@ import {
     InteractionResult as _InteractionResult,
 } from 'cvat-canvas/src/typescript/canvas';
 
+export function convertShapesForInteractor(shapes: InteractionResult[]): number[][] {
+    const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => {
+        if (!(index % 2)) { // 0, 2, 4
+            acc.push([
+                array[index],
+                array[index + 1],
+            ]);
+        }
+        return acc;
+    };
+
+    return shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points' && shape.button === 0)
+        .map((shape: InteractionResult): number[] => shape.points)
+        .flat().reduce(reducer, []);
+}
+
 export type InteractionData = _InteractionData;
 export type InteractionResult = _InteractionResult;
 

cvat-ui/src/icons.tsx

@@ -47,6 +47,7 @@ import SVGCubeIcon from './assets/cube-icon.svg';
 import SVGResetPerspectiveIcon from './assets/reset-perspective.svg';
 import SVGColorizeIcon from './assets/colorize-icon.svg';
 import SVGAITools from './assets/ai-tools-icon.svg';
+import SVGOpenCV from './assets/opencv.svg';
 
 export const CVATLogo = React.memo((): JSX.Element => <SVGCVATLogo />);
 export const AccountIcon = React.memo((): JSX.Element => <SVGAccountIcon />);
@@ -91,3 +92,4 @@ export const CubeIcon = React.memo((): JSX.Element => <SVGCubeIcon />);
 export const ResetPerspectiveIcon = React.memo((): JSX.Element => <SVGResetPerspectiveIcon />);
 export const AIToolsIcon = React.memo((): JSX.Element => <SVGAITools />);
 export const ColorizeIcon = React.memo((): JSX.Element => <SVGColorizeIcon />);
+export const OpenCVIcon = React.memo((): JSX.Element => <SVGOpenCV />);

cvat-ui/src/reducers/annotation-reducer.ts

@@ -1010,6 +1010,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
             };
         }
         case AnnotationActionTypes.INTERACT_WITH_CANVAS: {
+            const { activeInteractor, activeLabelID } = action.payload;
             return {
                 ...state,
                 annotations: {
@@ -1018,12 +1019,13 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                 },
                 drawing: {
                     ...state.drawing,
-                    activeInteractor: action.payload.activeInteractor,
-                    activeLabelID: action.payload.activeLabelID,
+                    activeInteractor,
+                    activeLabelID,
                 },
                 canvas: {
                     ...state.canvas,
-                    activeControl: ActiveControl.AI_TOOLS,
+                    activeControl: activeInteractor
+                        .type.startsWith('opencv') ? ActiveControl.OPENCV_TOOLS : ActiveControl.AI_TOOLS,
                 },
             };
         }

cvat-ui/src/reducers/interfaces.ts

-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
 import { ExtendedKeyMapOptions } from 'react-hotkeys';
 import { Canvas, RectDrawingMethod } from 'cvat-canvas-wrapper';
+import { IntelligentScissors } from 'utils/opencv-wrapper/intelligent-scissors';
 import { MutableRefObject } from 'react';
 
 export type StringObject = {
@@ -173,6 +174,7 @@ export interface Model {
     };
 }
 
+export type OpenCVTool = IntelligentScissors;
 export enum TaskStatus {
     ANNOTATION = 'annotation',
     REVIEW = 'validation',
@@ -330,6 +332,7 @@ export enum ActiveControl {
     EDIT = 'edit',
     OPEN_ISSUE = 'open_issue',
     AI_TOOLS = 'ai_tools',
+    OPENCV_TOOLS = 'opencv_tools',
 }
 
 export enum ShapeType {
@@ -403,7 +406,7 @@ export interface AnnotationState {
         frameAngles: number[];
     };
     drawing: {
-        activeInteractor?: Model;
+        activeInteractor?: Model | OpenCVTool;
         activeShapeType: ShapeType;
         activeRectDrawingMethod?: RectDrawingMethod;
         activeNumOfPoints?: number;

cvat-ui/src/utils/math.ts

-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
+
 export function clamp(value: number, min: number, max: number): number {
     return Math.max(Math.min(value, max), min);
 }
@@ -11,3 +12,28 @@ export function shift<T>(array: Array<T>, k: number): Array<T> {
     }
     return array;
 }
+
+export interface Point {
+    x: number;
+    y: number;
+}
+
+export function numberArrayToPoints(coordinates: number[]): Point[] {
+    return coordinates.reduce((acc: Point[], _: number, index: number): Point[] => {
+        if (index % 2) {
+            acc.push({
+                x: coordinates[index - 1],
+                y: coordinates[index],
+            });
+        }
+
+        return acc;
+    }, []);
+}
+
+export function pointsToNumberArray(points: Point[]): number[] {
+    return points.reduce((acc: number[], point: Point): number[] => {
+        acc.push(point.x, point.y);
+        return acc;
+    }, []);
+}

cvat-ui/src/utils/opencv-wrapper/intelligent-scissors.ts

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import { numberArrayToPoints, pointsToNumberArray, Point } from '../math';
+
+export interface IntelligentScissorsParams {
+    shape: {
+        shapeType: 'polygon' | 'polyline';
+    };
+    canvas: {
+        shapeType: 'points';
+        enableThreshold: boolean;
+        enableSliding: boolean;
+        allowRemoveOnlyLast: boolean;
+        minPosVertices: number;
+    };
+}
+
+export interface IntelligentScissors {
+    reset(): void;
+    run(points: number[], image: ImageData, offsetX: number, offsetY: number): number[];
+    params: IntelligentScissorsParams;
+}
+
+function applyOffset(points: Point[], offsetX: number, offsetY: number): Point[] {
+    return points.map(
+        (point: Point): Point => ({
+            x: point.x - offsetX,
+            y: point.y - offsetY,
+        }),
+    );
+}
+
+export default class IntelligentScissorsImplementation implements IntelligentScissors {
+    private cv: any;
+    private scissors: {
+        tool: any;
+        state: {
+            path: number[];
+            anchors: Record<
+            number,
+            {
+                point: Point;
+                start: number;
+            }
+            >; // point index : start index in path
+            image: any | null;
+        };
+    };
+
+    public constructor(cv: any) {
+        this.cv = cv;
+        this.reset();
+    }
+
+    public reset(): void {
+        if (this.scissors && this.scissors.tool) {
+            this.scissors.tool.delete();
+        }
+
+        this.scissors = {
+            // eslint-disable-next-line new-cap
+            tool: new this.cv.segmentation_IntelligentScissorsMB(),
+            state: {
+                path: [],
+                anchors: {},
+                image: null,
+            },
+        };
+
+        this.scissors.tool.setEdgeFeatureCannyParameters(32, 100);
+        this.scissors.tool.setGradientMagnitudeMaxLimit(200);
+    }
+
+    public run(coordinates: number[], image: ImageData, offsetX: number, offsetY: number): number[] {
+        if (!Array.isArray(coordinates)) {
+            throw new Error('Coordinates is expected to be an array');
+        }
+        if (!coordinates.length) {
+            throw new Error('At least one point is expected');
+        }
+        if (!(image instanceof ImageData)) {
+            throw new Error('Image is expected to be an instance of ImageData');
+        }
+
+        const { cv, scissors } = this;
+        const { tool, state } = scissors;
+
+        const points = applyOffset(numberArrayToPoints(coordinates), offsetX, offsetY);
+
+        if (points.length > 1) {
+            let matImage = null;
+            const contour = new cv.Mat();
+            const approx = new cv.Mat();
+
+            try {
+                const [prev, cur] = points.slice(-2);
+                const { x: prevX, y: prevY } = prev;
+                const { x: curX, y: curY } = cur;
+
+                const latestPointRemoved = points.length < Object.keys(state.anchors).length;
+                const latestPointReplaced = points.length === Object.keys(state.anchors).length;
+
+                if (latestPointRemoved) {
+                    for (const i of Object.keys(state.anchors).sort((a, b) => +b - +a)) {
+                        if (+i >= points.length) {
+                            state.path = state.path.slice(0, state.anchors[points.length].start);
+                            delete state.anchors[+i];
+                        }
+                    }
+
+                    return [...state.path];
+                }
+
+                matImage = cv.matFromImageData(image);
+
+                if (latestPointReplaced) {
+                    state.path = state.path.slice(0, state.anchors[points.length - 1].start);
+                    delete state.anchors[points.length - 1];
+                }
+
+                tool.applyImage(matImage);
+                tool.buildMap(new cv.Point(prevX, prevY));
+                tool.getContour(new cv.Point(curX, curY), contour);
+                cv.approxPolyDP(contour, approx, 2, false);
+
+                const pathSegment = [];
+                for (let row = 0; row < approx.rows; row++) {
+                    pathSegment.push(approx.intAt(row, 0) + offsetX, approx.intAt(row, 1) + offsetY);
+                }
+                state.anchors[points.length - 1] = {
+                    point: cur,
+                    start: state.path.length,
+                };
+                state.path.push(...pathSegment);
+            } finally {
+                if (matImage) {
+                    matImage.delete();
+                }
+
+                contour.delete();
+                approx.delete();
+            }
+        } else {
+            state.path.push(...pointsToNumberArray(applyOffset(points.slice(-1), -offsetX, -offsetY)));
+            state.anchors[0] = {
+                point: points[0],
+                start: 0,
+            };
+        }
+
+        return [...state.path];
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    public get type(): string {
+        return 'opencv_intelligent_scissors';
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    public get params(): IntelligentScissorsParams {
+        return {
+            shape: {
+                shapeType: 'polygon',
+            },
+            canvas: {
+                shapeType: 'points',
+                enableThreshold: true,
+                enableSliding: true,
+                allowRemoveOnlyLast: true,
+                minPosVertices: 1,
+            },
+        };
+    }
+}

cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts

+// Copyright (C) 2020-2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import getCore from 'cvat-core-wrapper';
+import waitFor from '../wait-for';
+
+import IntelligentScissorsImplementation, { IntelligentScissors } from './intelligent-scissors';
+
+const core = getCore();
+const baseURL = core.config.backendAPI.slice(0, -7);
+
+export interface Segmentation {
+    intelligentScissorsFactory: () => IntelligentScissors;
+}
+
+export class OpenCVWrapper {
+    private initialized: boolean;
+    private cv: any;
+
+    public constructor() {
+        this.initialized = false;
+        this.cv = null;
+    }
+
+    public async initialize(onProgress: (percent: number) => void): Promise<void> {
+        const response = await fetch(`${baseURL}/opencv/opencv.js`);
+        if (response.status !== 200) {
+            throw new Error(`Response status ${response.status}. ${response.statusText}`);
+        }
+
+        const contentLength = response.headers.get('Content-Length');
+        const { body } = response;
+
+        if (contentLength === null) {
+            throw new Error('Content length is null, but necessary');
+        }
+
+        if (body === null) {
+            throw new Error('Response body is null, but necessary');
+        }
+
+        const decoder = new TextDecoder('utf-8');
+        const reader = (body as ReadableStream<Uint8Array>).getReader();
+        let recieved = false;
+        let receivedLength = 0;
+        let decodedScript = '';
+
+        while (!recieved) {
+            // await in the loop is necessary here
+            // eslint-disable-next-line
+            const { done, value } = await reader.read();
+            recieved = done;
+
+            if (value instanceof Uint8Array) {
+                decodedScript += decoder.decode(value);
+                receivedLength += value.length;
+                const percentage = (receivedLength * 100) / +(contentLength as string);
+                onProgress(+percentage.toFixed(0));
+            }
+        }
+
+        // Inject opencv to DOM
+        // eslint-disable-next-line @typescript-eslint/no-implied-eval
+        const OpenCVConstructor = new Function(decodedScript);
+        OpenCVConstructor();
+
+        const global = window as any;
+        await waitFor(
+            100,
+            () =>
+                typeof global.cv !== 'undefined' && typeof global.cv.segmentation_IntelligentScissorsMB !== 'undefined',
+        );
+
+        this.cv = global.cv;
+        this.initialized = true;
+    }
+
+    public get isInitialized(): boolean {
+        return this.initialized;
+    }
+
+    public get segmentation(): Segmentation {
+        if (!this.initialized) {
+            throw new Error('Need to initialize OpenCV first');
+        }
+
+        return {
+            intelligentScissorsFactory: () => new IntelligentScissorsImplementation(this.cv),
+        };
+    }
+}
+
+export default new OpenCVWrapper();

cvat-ui/src/utils/wait-for.ts

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+export default function waitFor(frequencyHz: number, predicate: CallableFunction): Promise<void> {
+    return new Promise((resolve, reject) => {
+        if (typeof predicate !== 'function') {
+            reject(new Error(`Predicate must be a function, got ${typeof predicate}`));
+        }
+
+        const internalWait = (): void => {
+            let result = false;
+            try {
+                result = predicate();
+            } catch (error) {
+                reject(error);
+            }
+
+            if (result) {
+                resolve();
+            } else {
+                setTimeout(internalWait, 1000 / frequencyHz);
+            }
+        };
+
+        setTimeout(internalWait);
+    });
+}

cvat/apps/opencv/README.md

+# Updating OpenCV.js
+
+The latest version of OpenCV JavaScript library can be pulled from [OpenCV site](https://docs.opencv.org/master/opencv.js).
+
+To install it just push `opencv.js` to <b>cvat/apps/opencv/static/opencv/js</b>
+
+If develop locally, do not forget update static files after pushing `python manage.py collectstatic`

cvat/apps/opencv/__init__.py

+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/opencv/apps.py

+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from django.apps import AppConfig
+
+class OpencvConfig(AppConfig):
+    name = 'opencv'

cvat/apps/opencv/migrations/__init__.py

+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT

cvat/apps/opencv/urls.py

+
+# Copyright (C) 2018-2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+from django.urls import path
+from . import views
+
+urlpatterns = [
+    path('opencv.js', views.OpenCVLibrary)
+]

cvat/apps/opencv/views.py

+import os
+import glob
+from django.conf import settings
+from sendfile import sendfile
+
+def OpenCVLibrary(request):
+    dirname = os.path.join(settings.STATIC_ROOT, 'opencv', 'js')
+    pattern = os.path.join(dirname, 'opencv_*.js')
+    path = glob.glob(pattern)[0]
+    return sendfile(request, path)

cvat/settings/base.py

@@ -108,6 +108,7 @@ INSTALLED_APPS = [
     'cvat.apps.dataset_repo',
     'cvat.apps.restrictions',
     'cvat.apps.lambda_manager',
+    'cvat.apps.opencv',
     'django_rq',
     'compressor',
     'cacheops',

cvat/urls.py

@@ -38,5 +38,8 @@ if apps.is_installed('cvat.apps.log_viewer'):
 if apps.is_installed('cvat.apps.lambda_manager'):
     urlpatterns.append(path('', include('cvat.apps.lambda_manager.urls')))
 
+if apps.is_installed('cvat.apps.opencv'):
+    urlpatterns.append(path('opencv/', include('cvat.apps.opencv.urls')))
+
 if apps.is_installed('silk'):
     urlpatterns.append(path('profiler/', include('silk.urls')))

cvat_proxy/conf.d/cvat.conf.template

@@ -12,7 +12,7 @@ server {
     proxy_set_header        Host $http_host;
     proxy_pass_header       Set-Cookie;
 
-    location ~* /api/.*|git/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? {
+    location ~* /api/.*|git/.*|opencv/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? {
         proxy_pass              http://cvat:8080;
     }
 

mod_wsgi.conf

 LoadModule xsendfile_module /usr/lib/apache2/modules/mod_xsendfile.so
 XSendFile On
 XSendFilePath ${HOME}/data/
+XSendFilePath ${HOME}/static/