Histogram equalization feature (#3447)

* Added histogram equalization

* Fixed equalization memory leak

* Removed unused console.log

* Fixed eslint errors

* Fixed algorithm implementation in opencv control

* Fixed histogram equalization disabling

* Fixed eslint errors

* Removed outdated code and reworked cycles in functions

* Fixed forceUpdate flag disabling

* Fixed image bitmap creation

* Fixed running setState and imageModifier

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Ability to export/import tasks (<https://github.com/openvinotoolkit/cvat/pull/3056>)
 - Add a tutorial for semi-automatic/automatic annotation (<https://github.com/openvinotoolkit/cvat/pull/3124>)
 - Explicit "Done" button when drawing any polyshapes (<https://github.com/openvinotoolkit/cvat/pull/3417>)
+- Histogram equalization with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/3447>)
 
 ### Changed
 

cvat-canvas/src/typescript/canvasModel.ts

@@ -58,6 +58,7 @@ export interface Configuration {
     showProjections?: boolean;
     forceDisableEditing?: boolean;
     intelligentPolygonCrop?: boolean;
+    forceFrameUpdate?: boolean;
 }
 
 export interface DrawData {
@@ -392,8 +393,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
                 throw Error(`Canvas is busy. Action: ${this.data.mode}`);
             }
         }
-
-        if (frameData.number === this.data.imageID) {
+        if (frameData.number === this.data.imageID && !this.data.configuration.forceFrameUpdate) {
             this.data.zLayer = zLayer;
             this.data.objects = objectStates;
             this.notify(UpdateReasons.OBJECTS_UPDATED);
@@ -652,6 +652,10 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
             this.data.configuration.intelligentPolygonCrop = configuration.intelligentPolygonCrop;
         }
 
+        if (typeof configuration.forceFrameUpdate === 'boolean') {
+            this.data.configuration.forceFrameUpdate = configuration.forceFrameUpdate;
+        }
+
         this.notify(UpdateReasons.CONFIG_UPDATED);
     }
 

cvat-core/src/api.js

@@ -20,6 +20,7 @@ function build() {
     const { Project } = require('./project');
     const { Attribute, Label } = require('./labels');
     const MLModel = require('./ml-model');
+    const { FrameData } = require('./frames');
 
     const enums = require('./enums');
 
@@ -765,6 +766,7 @@ function build() {
             Comment,
             Issue,
             Review,
+            FrameData,
         },
     };
 

cvat-core/src/frames.js

@@ -124,6 +124,14 @@
             const result = await PluginRegistry.apiWrapper.call(this, FrameData.prototype.data, onServerRequest);
             return result;
         }
+
+        get imageData() {
+            return this._data.imageData;
+        }
+
+        set imageData(imageData) {
+            this._data.imageData = imageData;
+        }
     }
 
     FrameData.prototype.data.implementation = async function (onServerRequest) {

cvat-ui/src/actions/annotation-actions.ts

@@ -689,7 +689,8 @@ export function getPredictionsAsync(): ThunkAction {
     };
 }
 
-export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameStep?: number): ThunkAction {
+export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameStep?: number,
+    forceUpdate?: boolean): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
         const state: CombinedState = getStore().getState();
         const { instance: job } = state.annotation.job;
@@ -700,7 +701,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
                 throw Error(`Required frame ${toFrame} is out of the current job`);
             }
 
-            if (toFrame === frame) {
+            if (toFrame === frame && !forceUpdate) {
                 dispatch({
                     type: AnnotationActionTypes.CHANGE_FRAME_SUCCESS,
                     payload: {
@@ -719,7 +720,6 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
 
                 return;
             }
-
             // Start async requests
             dispatch({
                 type: AnnotationActionTypes.CHANGE_FRAME,

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/opencv-control.tsx

@@ -6,7 +6,7 @@ import React from 'react';
 import { connect } from 'react-redux';
 import { Row, Col } from 'antd/lib/grid';
 import Popover from 'antd/lib/popover';
-import Icon, { ScissorOutlined } from '@ant-design/icons';
+import Icon, { AreaChartOutlined, ScissorOutlined } from '@ant-design/icons';
 import Text from 'antd/lib/typography/Text';
 import Tabs from 'antd/lib/tabs';
 import Button from 'antd/lib/button';
@@ -26,9 +26,11 @@ import {
     fetchAnnotationsAsync,
     updateAnnotationsAsync,
     createAnnotationsAsync,
+    changeFrameAsync,
 } from 'actions/annotation-actions';
 import LabelSelector from 'components/label-selector/label-selector';
 import CVATTooltip from 'components/common/cvat-tooltip';
+import { ImageProcessing } from 'utils/opencv-wrapper/opencv-interfaces';
 import withVisibilityHandling from './handle-popover-visibility';
 
 interface Props {
@@ -39,6 +41,7 @@ interface Props {
     states: any[];
     frame: number;
     curZOrder: number;
+    frameData: any;
 }
 
 interface DispatchToProps {
@@ -46,6 +49,7 @@ interface DispatchToProps {
     updateAnnotations(statesToUpdate: any[]): void;
     createAnnotations(sessionInstance: any, frame: number, statesToCreate: any[]): void;
     fetchAnnotations(): void;
+    changeFrame(toFrame: number, fillBuffer?: boolean, frameStep?: number, forceUpdate?: boolean):void;
 }
 
 interface State {
@@ -53,6 +57,12 @@ interface State {
     initializationError: boolean;
     initializationProgress: number;
     activeLabelID: number;
+    activeImageModifiers: ImageModifier[];
+}
+
+interface ImageModifier {
+    modifier: ImageProcessing,
+    alias: string
 }
 
 const core = getCore();
@@ -68,7 +78,7 @@ function mapStateToProps(state: CombinedState): Props {
             job: { instance: jobInstance, labels },
             canvas: { activeControl, instance: canvasInstance },
             player: {
-                frame: { number: frame },
+                frame: { number: frame, data: frameData },
             },
         },
     } = state;
@@ -81,6 +91,7 @@ function mapStateToProps(state: CombinedState): Props {
         labels,
         states,
         frame,
+        frameData,
     };
 }
 
@@ -89,26 +100,32 @@ const mapDispatchToProps = {
     updateAnnotations: updateAnnotationsAsync,
     fetchAnnotations: fetchAnnotationsAsync,
     createAnnotations: createAnnotationsAsync,
+    changeFrame: changeFrameAsync,
 };
 
 class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps, State> {
     private activeTool: IntelligentScissors | null;
+    private canvasForceUpdateWasEnabled: boolean;
 
     public constructor(props: Props & DispatchToProps) {
         super(props);
         const { labels } = props;
         this.activeTool = null;
+        this.canvasForceUpdateWasEnabled = false;
+
         this.state = {
             libraryInitialized: openCVWrapper.isInitialized,
             initializationError: false,
             initializationProgress: -1,
             activeLabelID: labels.length ? labels[0].id : null,
+            activeImageModifiers: [],
         };
     }
 
     public componentDidMount(): void {
         const { canvasInstance } = this.props;
         canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().addEventListener('canvas.setup', this.runImageModifier);
     }
 
     public componentDidUpdate(prevProps: Props): void {
@@ -124,6 +141,7 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
     public componentWillUnmount(): void {
         const { canvasInstance } = this.props;
         canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
+        canvasInstance.html().removeEventListener('canvas.setup', this.runImageModifier);
     }
 
     private interactionListener = async (e: Event): Promise<void> => {
@@ -173,6 +191,42 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         }
     };
 
+    private runImageModifier = async ():Promise<void> => {
+        const { activeImageModifiers } = this.state;
+        const {
+            frameData, states, curZOrder, canvasInstance, frame,
+        } = this.props;
+        try {
+            if (activeImageModifiers.length !== 0 && activeImageModifiers[0].modifier.currentProcessedImage !== frame) {
+                this.enableCanvasForceUpdate();
+                const canvas: HTMLCanvasElement | undefined = window.document.getElementById('cvat_canvas_background') as
+                    | HTMLCanvasElement
+                    | undefined;
+                if (!canvas) {
+                    throw new Error('Element #cvat_canvas_background was not found');
+                }
+                const { width, height } = canvas;
+                const context = canvas.getContext('2d');
+                if (!context) {
+                    throw new Error('Canvas context is empty');
+                }
+                const imageData = context.getImageData(0, 0, width, height);
+                const newImageData = activeImageModifiers.reduce((oldImageData, activeImageModifier) =>
+                    activeImageModifier.modifier.processImage(oldImageData, frame), imageData);
+                const imageBitmap = await createImageBitmap(newImageData);
+                frameData.imageData = imageBitmap;
+                canvasInstance.setup(frameData, states, curZOrder);
+            }
+        } catch (error) {
+            notification.error({
+                description: error.toString(),
+                message: 'OpenCV.js processing error occured',
+            });
+        } finally {
+            this.disableCanvasForceUpdate();
+        }
+    };
+
     private async runCVAlgorithm(pressedPoints: number[], threshold: number): Promise<number[]> {
         // Getting image data
         const canvas: HTMLCanvasElement | undefined = window.document.getElementById('cvat_canvas_background') as
@@ -215,6 +269,45 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         return points;
     }
 
+    private imageModifier(alias: string): ImageProcessing|null {
+        const { activeImageModifiers } = this.state;
+        return activeImageModifiers.find((imageModifier) => imageModifier.alias === alias)?.modifier || null;
+    }
+
+    private disableImageModifier(alias: string):void {
+        const { activeImageModifiers } = this.state;
+        const index = activeImageModifiers.findIndex((imageModifier) => imageModifier.alias === alias);
+        if (index !== -1) {
+            activeImageModifiers.splice(index, 1);
+            this.setState({
+                activeImageModifiers: [...activeImageModifiers],
+            });
+        }
+    }
+
+    private enableImageModifier(modifier: ImageProcessing, alias: string): void{
+        this.setState((prev: State) => ({
+            ...prev,
+            activeImageModifiers: [...prev.activeImageModifiers, { modifier, alias }],
+        }), () => {
+            this.runImageModifier();
+        });
+    }
+
+    private enableCanvasForceUpdate():void{
+        const { canvasInstance } = this.props;
+        canvasInstance.configure({ forceFrameUpdate: true });
+        this.canvasForceUpdateWasEnabled = true;
+    }
+
+    private disableCanvasForceUpdate():void{
+        if (this.canvasForceUpdateWasEnabled) {
+            const { canvasInstance } = this.props;
+            canvasInstance.configure({ forceFrameUpdate: false });
+            this.canvasForceUpdateWasEnabled = false;
+        }
+    }
+
     private renderDrawingContent(): JSX.Element {
         const { activeLabelID } = this.state;
         const { labels, canvasInstance, onInteractionStart } = this.props;
@@ -254,6 +347,36 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
         );
     }
 
+    private renderImageContent():JSX.Element {
+        return (
+            <Row justify='start'>
+                <Col>
+                    <CVATTooltip title='Histogram equalization' className='cvat-opencv-image-tool'>
+                        <Button
+                            className={this.imageModifier('histogram') ? 'cvat-opencv-image-tool-active' : ''}
+                            onClick={(e: React.MouseEvent<HTMLElement>) => {
+                                const modifier = this.imageModifier('histogram');
+                                if (!modifier) {
+                                    this.enableImageModifier(openCVWrapper.imgproc.hist(), 'histogram');
+                                } else {
+                                    const button = e.target as HTMLElement;
+                                    button.blur();
+                                    this.disableImageModifier('histogram');
+                                    const { changeFrame } = this.props;
+                                    const { frame } = this.props;
+                                    this.enableCanvasForceUpdate();
+                                    changeFrame(frame, false, 1, true);
+                                }
+                            }}
+                        >
+                            <AreaChartOutlined />
+                        </Button>
+                    </CVATTooltip>
+                </Col>
+            </Row>
+        );
+    }
+
     private renderContent(): JSX.Element {
         const { libraryInitialized, initializationProgress, initializationError } = this.state;
 
@@ -271,7 +394,9 @@ class OpenCVControlComponent extends React.PureComponent<Props & DispatchToProps
                         <Tabs.TabPane key='drawing' tab='Drawing' className='cvat-opencv-control-tabpane'>
                             {this.renderDrawingContent()}
                         </Tabs.TabPane>
-                        <Tabs.TabPane disabled key='image' tab='Image' className='cvat-opencv-control-tabpane' />
+                        <Tabs.TabPane key='image' tab='Image' className='cvat-opencv-control-tabpane'>
+                            {this.renderImageContent()}
+                        </Tabs.TabPane>
                     </Tabs>
                 ) : (
                     <>

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -228,6 +228,15 @@
     }
 }
 
+.cvat-opencv-image-tool {
+    @extend .cvat-opencv-drawing-tool;
+}
+
+.cvat-opencv-image-tool-active {
+    color: #40a9ff;
+    border-color: #40a9ff;
+}
+
 .cvat-setup-tag-popover-content,
 .cvat-draw-shape-popover-content {
     padding: $grid-unit-size;

cvat-ui/src/utils/opencv-wrapper/histogram-equalization.ts

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import { ImageProcessing } from './opencv-interfaces';
+
+export interface HistogramEqualization extends ImageProcessing{
+    processImage: (src:ImageData, frameNumber: number)=>ImageData;
+}
+
+interface HashedImage{
+    frameNumber: number,
+    frameData: ImageData,
+    timestamp: number,
+}
+
+export default class HistogramEqualizationImplementation implements HistogramEqualization {
+    private readonly bufferSize: number = 20;
+    private cv:any;
+    private histHash: HashedImage[];
+    public currentProcessedImage: number | undefined;
+
+    constructor(cv:any) {
+        this.cv = cv;
+        this.histHash = [];
+    }
+
+    public processImage(src:ImageData, frameNumber: number) : ImageData {
+        const hashedFrame = this.hashedFrame(frameNumber);
+        if (!hashedFrame) {
+            const { cv } = this;
+            let matImage = null;
+            const RGBImage = new cv.Mat();
+            const YUVImage = new cv.Mat();
+            const RGBDist = new cv.Mat();
+            const YUVDist = new cv.Mat();
+            const RGBADist = new cv.Mat();
+            let channels = new cv.MatVector();
+            const equalizedY = new cv.Mat();
+            try {
+                this.currentProcessedImage = frameNumber;
+                matImage = cv.matFromImageData(src);
+                cv.cvtColor(matImage, RGBImage, cv.COLOR_RGBA2RGB, 0);
+                cv.cvtColor(RGBImage, YUVImage, cv.COLOR_RGB2YUV, 0);
+                cv.split(YUVImage, channels);
+                const [Y, U, V] = [channels.get(0), channels.get(1), channels.get(2)];
+                channels.delete();
+                channels = null;
+                cv.equalizeHist(Y, equalizedY);
+                Y.delete();
+                channels = new cv.MatVector();
+                channels.push_back(equalizedY); equalizedY.delete();
+                channels.push_back(U); U.delete();
+                channels.push_back(V); V.delete();
+                cv.merge(channels, YUVDist);
+                cv.cvtColor(YUVDist, RGBDist, cv.COLOR_YUV2RGB, 0);
+                cv.cvtColor(RGBDist, RGBADist, cv.COLOR_RGB2RGBA, 0);
+                const arr = new Uint8ClampedArray(RGBADist.data, RGBADist.cols, RGBADist.rows);
+                const imgData = new ImageData(arr, src.width, src.height);
+                this.hashFrame(imgData, frameNumber);
+                return imgData;
+            } catch (e) {
+                console.log('Histogram equalization error', e);
+                return src;
+            } finally {
+                if (matImage) matImage.delete();
+                if (channels) channels.delete();
+                RGBImage.delete();
+                YUVImage.delete();
+                RGBDist.delete();
+                YUVDist.delete();
+                RGBADist.delete();
+            }
+        } else {
+            this.currentProcessedImage = frameNumber;
+            return hashedFrame;
+        }
+    }
+
+    private hashedFrame(frameNumber: number): ImageData|null {
+        const hashed = this.histHash.find((_hashed) => _hashed.frameNumber === frameNumber);
+        if (hashed) {
+            hashed.timestamp = Date.now();
+        }
+        return hashed?.frameData || null;
+    }
+
+    private hashFrame(frameData:ImageData, frameNumber:number):void{
+        if (this.histHash.length >= this.bufferSize) {
+            const leastRecentlyUsed = this.histHash[0];
+            const currentTimestamp = Date.now();
+            let diff = currentTimestamp - leastRecentlyUsed.timestamp;
+            let leastIndex = 0;
+            for (let i = 1; i < this.histHash.length; i++) {
+                const currentDiff = currentTimestamp - this.histHash[i].timestamp;
+                if (currentDiff > diff) {
+                    diff = currentDiff;
+                    leastIndex = i;
+                }
+            }
+            this.histHash.splice(leastIndex, 1);
+        }
+        this.histHash.push({
+            frameData,
+            frameNumber,
+            timestamp: Date.now(),
+        });
+    }
+}

cvat-ui/src/utils/opencv-wrapper/opencv-interfaces.ts

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+export interface ImageProcessing {
+    processImage: (src: ImageData, frameNumber: number) => ImageData;
+    currentProcessedImage: number|undefined
+}

cvat-ui/src/utils/opencv-wrapper/opencv-wrapper.ts

@@ -4,6 +4,7 @@
 
 import getCore from 'cvat-core-wrapper';
 import waitFor from '../wait-for';
+import HistogramEqualizationImplementation, { HistogramEqualization } from './histogram-equalization';
 
 import IntelligentScissorsImplementation, { IntelligentScissors } from './intelligent-scissors';
 
@@ -14,6 +15,10 @@ export interface Segmentation {
     intelligentScissorsFactory: () => IntelligentScissors;
 }
 
+export interface ImgProc {
+    hist: () => HistogramEqualization
+}
+
 export class OpenCVWrapper {
     private initialized: boolean;
     private cv: any;
@@ -89,6 +94,15 @@ export class OpenCVWrapper {
             intelligentScissorsFactory: () => new IntelligentScissorsImplementation(this.cv),
         };
     }
+
+    public get imgproc(): ImgProc {
+        if (!this.initialized) {
+            throw new Error('Need to initialize OpenCV first');
+        }
+        return {
+            hist: () => new HistogramEqualizationImplementation(this.cv),
+        };
+    }
 }
 
 export default new OpenCVWrapper();

site/content/en/docs/manual/advanced/opencv-tools.md

@@ -38,3 +38,21 @@ displayed as a red square which is tied to the cursor.
 
 - Once all the points are placed, you can complete the creation of the object by clicking on the icon or clicking `N`.
   As a result, a polygon will be created (read more about the polygons in the [annotation with polygons](/docs/manual/advanced/annotation-with-polygons/)).
+
+### Histogram Equalization
+
+Histogram equalization is an CV method that improves contrast in an image in order to stretch out the intensity range.
+This method usually increases the global contrast of images when its usable data
+is represented by close contrast values.
+It is useful in images with backgrounds and foregrounds that are both bright or both dark.
+
+- First, select the image tab and then click on `histogram equalization` button.
+
+  ![](/images/image221.jpg)
+
+- Then contrast of current frame will be improved.
+  If you change frame, it will be equalized too.
+  You can disable equalization by clicking `histogram equalization` button again.
+
+  ![](/images/image222.jpg)
+