Support of context images for 2D tasks (#3122)

CHANGELOG.md

@@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
--
+- Support of context images for 2D image tasks (<https://github.com/openvinotoolkit/cvat/pull/3122>)
 
 ### Changed
 
--
+- Updated manifest format, added meta with related images (<https://github.com/openvinotoolkit/cvat/pull/3122>)
 
 ### Deprecated
 
@@ -111,7 +111,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed filters select overflow (<https://github.com/openvinotoolkit/cvat/pull/2614>)
 - Fixed tasks in project auto annotation (<https://github.com/openvinotoolkit/cvat/pull/2725>)
 - Cuboids are missed in annotations statistics (<https://github.com/openvinotoolkit/cvat/pull/2704>)
-- The list of files attached to the task is not displayed (<https://github.com/openvinotoolkit/cvat/pul
+- The list of files attached to the task is not displayed (<https://github.com/openvinotoolkit/cvat/pull/2706>)
 - A couple of css-related issues (top bar disappear, wrong arrow position on collapse elements) (<https://github.com/openvinotoolkit/cvat/pull/2736>)
 - Issue with point region doesn't work in Firefox (<https://github.com/openvinotoolkit/cvat/pull/2727>)
 - Fixed cuboid perspective change (<https://github.com/openvinotoolkit/cvat/pull/2733>)

cvat-core/package-lock.json

 {
   "name": "cvat-core",
-  "version": "3.12.1",
+  "version": "3.13.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-core/package.json

 {
   "name": "cvat-core",
-  "version": "3.12.1",
+  "version": "3.13.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/frames.js

@@ -19,7 +19,15 @@
      */
     class FrameData {
         constructor({
-            width, height, name, taskID, frameNumber, startFrame, stopFrame, decodeForward,
+            width,
+            height,
+            name,
+            taskID,
+            frameNumber,
+            startFrame,
+            stopFrame,
+            decodeForward,
+            has_related_context: hasRelatedContext,
         }) {
             Object.defineProperties(
                 this,
@@ -72,6 +80,18 @@
                         value: frameNumber,
                         writable: false,
                     },
+                    /**
+                     * True if some context images are associated with this frame
+                     * @name hasRelatedContext
+                     * @type {boolean}
+                     * @memberof module:API.cvat.classes.FrameData
+                     * @readonly
+                     * @instance
+                     */
+                    hasRelatedContext: {
+                        value: hasRelatedContext,
+                        writable: false,
+                    },
                     startFrame: {
                         value: startFrame,
                         writable: false,

cvat-core/src/server-proxy.js

@@ -756,11 +756,7 @@
                         },
                     );
                 } catch (errorData) {
-                    const code = errorData.response ? errorData.response.status : errorData.code;
-                    throw new ServerError(
-                        `Could not get Image Context of the frame for the task ${tid} from the server`,
-                        code,
-                    );
+                    throw generateError(errorData);
                 }
 
                 return response.data;

cvat-ui/package-lock.json

 {
   "name": "cvat-ui",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-ui/package.json

 {
   "name": "cvat-ui",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/annotation-actions.ts

@@ -196,6 +196,8 @@ export enum AnnotationActionTypes {
     GET_PREDICTIONS_SUCCESS = 'GET_PREDICTIONS_SUCCESS',
     HIDE_SHOW_CONTEXT_IMAGE = 'HIDE_SHOW_CONTEXT_IMAGE',
     GET_CONTEXT_IMAGE = 'GET_CONTEXT_IMAGE',
+    GET_CONTEXT_IMAGE_SUCCESS = 'GET_CONTEXT_IMAGE_SUCCESS',
+    GET_CONTEXT_IMAGE_FAILED = 'GET_CONTEXT_IMAGE_FAILED',
 }
 
 export function saveLogsAsync(): ThunkAction {
@@ -715,6 +717,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
                         number: state.annotation.player.frame.number,
                         data: state.annotation.player.frame.data,
                         filename: state.annotation.player.frame.filename,
+                        hasRelatedContext: state.annotation.player.frame.hasRelatedContext,
                         delay: state.annotation.player.frame.delay,
                         changeTime: state.annotation.player.frame.changeTime,
                         states: state.annotation.annotations.states,
@@ -766,6 +769,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
                     number: toFrame,
                     data,
                     filename: data.filename,
+                    hasRelatedContext: data.hasRelatedContext,
                     states,
                     minZ,
                     maxZ,
@@ -1031,6 +1035,7 @@ export function getJobAsync(tid: number, jid: number, initialFrame: number, init
                     states,
                     frameNumber,
                     frameFilename: frameData.filename,
+                    frameHasRelatedContext: frameData.hasRelatedContext,
                     frameData,
                     colors,
                     filters,
@@ -1636,31 +1641,23 @@ export function getContextImage(): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
         const state: CombinedState = getStore().getState();
         const { instance: job } = state.annotation.job;
-        const { frame, contextImage } = state.annotation.player;
+        const { number: frameNumber } = state.annotation.player.frame;
 
         try {
-            const context = await job.frames.contextImage(job.task.id, frame.number);
-            const loaded = true;
-            const contextImageHide = contextImage.hidden;
             dispatch({
                 type: AnnotationActionTypes.GET_CONTEXT_IMAGE,
-                payload: {
-                    context,
-                    loaded,
-                    contextImageHide,
-                },
+                payload: {},
+            });
+
+            const contextImageData = await job.frames.contextImage(job.task.id, frameNumber);
+            dispatch({
+                type: AnnotationActionTypes.GET_CONTEXT_IMAGE_SUCCESS,
+                payload: { contextImageData },
             });
         } catch (error) {
-            const context = '';
-            const loaded = true;
-            const contextImageHide = contextImage.hidden;
             dispatch({
-                type: AnnotationActionTypes.GET_CONTEXT_IMAGE,
-                payload: {
-                    context,
-                    loaded,
-                    contextImageHide,
-                },
+                type: AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED,
+                payload: { error },
             });
         }
     };

cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx

@@ -18,6 +18,7 @@ import getCore from 'cvat-core-wrapper';
 import consts from 'consts';
 import CVATTooltip from 'components/common/cvat-tooltip';
 import ImageSetupsContent from './image-setups-content';
+import ContextImage from '../standard-workspace/context-image/context-image';
 
 const cvat = getCore();
 
@@ -773,12 +774,12 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
             maxZLayer,
             curZLayer,
             minZLayer,
-            onSwitchZLayer,
-            onAddZLayer,
             keyMap,
             switchableAutomaticBordering,
             automaticBordering,
             onSwitchAutomaticBordering,
+            onSwitchZLayer,
+            onAddZLayer,
         } = this.props;
 
         const preventDefault = (event: KeyboardEvent | undefined): void => {
@@ -817,6 +818,8 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
                     }}
                 />
 
+                <ContextImage />
+
                 <Dropdown trigger='click' placement='topCenter' overlay={<ImageSetupsContent />}>
                     <UpOutlined className='cvat-canvas-image-setups-trigger' />
                 </Dropdown>

cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx

@@ -14,20 +14,16 @@ import { Workspace } from 'reducers/interfaces';
 import {
     CAMERA_ACTION, Canvas3d, MouseInteraction, ViewType,
 } from 'cvat-canvas3d-wrapper';
-import ContextImage from '../standard3D-workspace/context-image/context-image';
-import CVATTooltip from '../../common/cvat-tooltip';
+import ContextImage from 'components/annotation-page/standard-workspace/context-image/context-image';
+import CVATTooltip from 'components/common/cvat-tooltip';
 
 interface Props {
     canvasInstance: Canvas3d;
     jobInstance: any;
     frameData: any;
     curZLayer: number;
-    contextImageHide: boolean;
-    loaded: boolean;
-    data: string;
     annotations: any[];
     onSetupCanvas: () => void;
-    getContextImage(): void;
     onResetCanvas(): void;
     workspace: Workspace;
     animateID: any;
@@ -119,9 +115,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => {
     const sideView = useRef<HTMLDivElement | null>(null);
     const frontView = useRef<HTMLDivElement | null>(null);
 
-    const {
-        frameData, contextImageHide, getContextImage, loaded, data, annotations, curZLayer,
-    } = props;
+    const { frameData, annotations, curZLayer } = props;
 
     const onCanvasSetup = (): void => {
         const { onSetupCanvas } = props;
@@ -345,13 +339,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => {
 
     return (
         <Layout.Content className='cvat-canvas3d-fullsize' id='canvas3d-container'>
-            <ContextImage
-                frame={frameData}
-                contextImageHide={contextImageHide}
-                getContextImage={getContextImage}
-                loaded={loaded}
-                data={data}
-            />
+            <ContextImage />
             <ResizableBox
                 className='cvat-resizable'
                 width={Infinity}

cvat-ui/src/components/annotation-page/standard-workspace/context-image/context-image.tsx

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import React, { useEffect, useState } from 'react';
+import { notification } from 'antd';
+import { useDispatch, useSelector } from 'react-redux';
+import { QuestionCircleOutlined, ShrinkOutlined } from '@ant-design/icons';
+import Spin from 'antd/lib/spin';
+import Image from 'antd/lib/image';
+
+import { CombinedState } from 'reducers/interfaces';
+import { hideShowContextImage, getContextImage } from 'actions/annotation-actions';
+import CVATTooltip from 'components/common/cvat-tooltip';
+
+export default function ContextImage(): JSX.Element | null {
+    const dispatch = useDispatch();
+    const { number: frame, hasRelatedContext } = useSelector((state: CombinedState) => state.annotation.player.frame);
+    const { data: contextImageData, hidden: contextImageHidden, fetching: contextImageFetching } = useSelector(
+        (state: CombinedState) => state.annotation.player.contextImage,
+    );
+    const [requested, setRequested] = useState(false);
+
+    useEffect(() => {
+        if (requested) {
+            setRequested(false);
+        }
+    }, [frame]);
+
+    useEffect(() => {
+        if (hasRelatedContext && !contextImageHidden && !requested) {
+            dispatch(getContextImage());
+            setRequested(true);
+        }
+    }, [contextImageHidden, requested, hasRelatedContext]);
+
+    if (!hasRelatedContext) {
+        return null;
+    }
+
+    return (
+        <div className='cvat-context-image-wrapper' {...(contextImageHidden ? { style: { width: '32px' } } : {})}>
+            <div className='cvat-context-image-wrapper-header' />
+            {contextImageFetching ? <Spin size='small' /> : null}
+            {contextImageHidden ? (
+                <CVATTooltip title='A context image is available'>
+                    <QuestionCircleOutlined
+                        className='cvat-context-image-switcher'
+                        onClick={() => dispatch(hideShowContextImage(false))}
+                    />
+                </CVATTooltip>
+            ) : (
+                <>
+                    <ShrinkOutlined
+                        className='cvat-context-image-switcher'
+                        onClick={() => dispatch(hideShowContextImage(true))}
+                    />
+                    <Image
+                        {...(contextImageData ? { src: contextImageData } : {})}
+                        onError={() => {
+                            notification.error({
+                                message: 'Could not display context image',
+                                description: `Source is  ${
+                                    contextImageData === null ? 'empty' : contextImageData.slice(0, 100)
+                                }`,
+                            });
+                        }}
+                        className='cvat-context-image'
+                    />
+                </>
+            )}
+        </div>
+    );
+}

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx

@@ -41,7 +41,7 @@ export function ExtraControlsControl(): JSX.Element {
         >
             <SmallDashOutlined
                 style={{ visibility: hasChildren ? 'visible' : 'hidden' }}
-                className='cvat-extra-controls-control'
+                className='cvat-extra-controls-control cvat-antd-icon-control'
             />
         </Popover>
     );

cvat-ui/src/components/annotation-page/standard-workspace/styles.scss

@@ -8,6 +8,55 @@
     height: 100%;
 }
 
+.cvat-context-image-wrapper {
+    height: auto;
+    width: $grid-unit-size * 32;
+    position: absolute;
+    top: $grid-unit-size;
+    right: $grid-unit-size;
+    z-index: 100;
+    background: black;
+    display: flex;
+    flex-direction: column;
+    justify-content: space-between;
+    user-select: none;
+
+    > .cvat-context-image-wrapper-header {
+        height: $grid-unit-size * 4;
+        width: 100%;
+        z-index: 101;
+        background: rgba(0, 0, 0, 0.2);
+        position: absolute;
+        top: 0;
+        left: 0;
+    }
+
+    > .ant-image {
+        margin: $grid-unit-size / 2;
+    }
+
+    > span {
+        position: absolute;
+        font-size: 18px;
+        top: 7px;
+        right: 7px;
+        z-index: 102;
+        color: white;
+
+        &:hover {
+            > svg {
+                transform: scale(1.2);
+            }
+        }
+    }
+}
+
+.cvat-context-image {
+    width: 100%;
+    height: auto;
+    display: block;
+}
+
 .cvat-objects-sidebar-sider {
     top: 0;
     right: 0;
@@ -56,8 +105,7 @@
 .cvat-issue-control,
 .cvat-tools-control,
 .cvat-extra-controls-control,
-.cvat-opencv-control,
-.cvat-context-image-control {
+.cvat-opencv-control {
     border-radius: 3.3px;
     transform: scale(0.65);
     padding: 2px;
@@ -76,7 +124,7 @@
     }
 }
 
-.cvat-extra-controls-control {
+.cvat-antd-icon-control {
     > svg {
         width: 40px;
         height: 40px;

cvat-ui/src/components/annotation-page/standard3D-workspace/context-image/context-image.tsx

-// Copyright (C) 2021 Intel Corporation
-//
-// SPDX-License-Identifier: MIT
-
-import React, { useEffect } from 'react';
-
-interface Props {
-    frame: number;
-    contextImageHide: boolean;
-    loaded: boolean;
-    data: string;
-    getContextImage(): void;
-}
-
-export default function ContextImage(props: Props): JSX.Element {
-    const {
-        contextImageHide, loaded, data, getContextImage,
-    } = props;
-
-    useEffect(() => {
-        if (!contextImageHide && !loaded) {
-            getContextImage();
-        }
-    }, [contextImageHide, loaded]);
-
-    if (!contextImageHide && data !== '') {
-        return (
-            <div className='cvat-contextImage'>
-                <img src={data} alt='Context not available' className='cvat-contextImage-show' />
-            </div>
-        );
-    }
-    return null;
-}

cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx

@@ -9,20 +9,15 @@ import { Canvas3d as Canvas } from 'cvat-canvas3d-wrapper';
 import CursorControl from './cursor-control';
 import MoveControl from './move-control';
 import DrawCuboidControl from './draw-cuboid-control';
-import PhotoContextControl from './photo-context';
 
 interface Props {
     canvasInstance: Canvas;
     activeControl: ActiveControl;
     normalizedKeyMap: Record<string, string>;
-    contextImageHide: boolean;
-    hideShowContextImage: (hidden: boolean) => void;
 }
 
 export default function ControlsSideBarComponent(props: Props): JSX.Element {
-    const {
-        canvasInstance, activeControl, normalizedKeyMap, contextImageHide, hideShowContextImage,
-    } = props;
+    const { canvasInstance, activeControl, normalizedKeyMap } = props;
 
     return (
         <Layout.Sider className='cvat-canvas-controls-sidebar' theme='light' width={44}>
@@ -37,12 +32,6 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                 canvasInstance={canvasInstance}
                 isDrawing={activeControl === ActiveControl.DRAW_CUBOID}
             />
-            <PhotoContextControl
-                canvasInstance={canvasInstance}
-                activeControl={activeControl}
-                contextImageHide={contextImageHide}
-                hideShowContextImage={hideShowContextImage}
-            />
         </Layout.Sider>
     );
 }

cvat-ui/src/components/annotation-page/standard3D-workspace/controls-side-bar/photo-context.tsx

-// Copyright (C) 2021 Intel Corporation
-//
-// SPDX-License-Identifier: MIT
-
-import React from 'react';
-import CameraIcon from '@ant-design/icons/CameraOutlined';
-
-import CVATTooltip from 'components/common/cvat-tooltip';
-import { Canvas3d as Canvas } from 'cvat-canvas3d-wrapper';
-import { ActiveControl } from 'reducers/interfaces';
-
-interface Props {
-    canvasInstance: Canvas;
-    activeControl: ActiveControl;
-    hideShowContextImage: (hidden: boolean) => void;
-    contextImageHide: boolean;
-}
-
-function PhotoContextControl(props: Props): JSX.Element {
-    const { activeControl, contextImageHide, hideShowContextImage } = props;
-
-    return (
-        <CVATTooltip title='Photo context show/hide' placement='right'>
-            <CameraIcon
-                className={`cvat-context-image-control
-    cvat-control-side-bar-icon-size ${
-        activeControl === ActiveControl.PHOTO_CONTEXT ? 'cvat-active-canvas-control' : ''
-        }`}
-                onClick={(): void => {
-                    hideShowContextImage(!contextImageHide);
-                }}
-            />
-        </CVATTooltip>
-    );
-}
-
-export default React.memo(PhotoContextControl);

cvat-ui/src/components/annotation-page/standard3D-workspace/styles.scss

@@ -4,173 +4,12 @@
 
 @import 'base.scss';
 
-.cvat-standard-workspace.ant-layout {
-    height: 100%;
-}
-
-.cvat-contextImage {
-    width: $grid-unit-size * 32;
-    position: absolute;
-    background: $border-color-3;
-    top: $grid-unit-size;
-    right: $grid-unit-size;
-    z-index: 100;
-    border-radius: $grid-unit-size;
-    border: 1px solid $border-color-3;
-    display: flex;
-    flex-direction: column;
-    justify-content: space-between;
-    padding: $grid-unit-size/2;
-}
-
-.cvat-contextImage-show {
-    max-width: 100%;
-    max-height: 100%;
-}
-
-.cvat-contextImage-loading {
-    text-align: center;
-}
-
-.cvat-objects-sidebar-filter-input {
-    width: calc(100% - 35px);
-}
-
-.cvat-objects-sidebar-sider {
-    top: 0;
-    right: 0;
-    left: auto;
-    background-color: $background-color-2;
-    border-left: 1px solid $border-color-1;
-    border-bottom: 1px solid $border-color-1;
-    border-radius: $grid-unit-size/2 0 0 $grid-unit-size/2;
-    z-index: 2;
-}
-
-.cvat-objects-sidebar {
-    height: 100%;
-}
-
-.cvat-rotate-canvas-controls-right > svg {
-    transform: scaleX(-1);
-}
-
-.cvat-canvas-controls-sidebar {
-    background-color: $background-color-2;
-    border-right: 1px solid $border-color-1;
-
-    > div {
-        > i {
-            border-radius: 3.3px;
-            transform: scale(0.65);
-            padding: $grid-unit-size/4;
-
-            &:hover {
-                background: $header-color;
-                transform: scale(0.75);
-            }
-
-            &:active {
-                transform: scale(0.65);
-            }
-
-            > svg {
-                transform: scale(0.8);
-            }
-        }
-    }
-}
-
-.cvat-active-canvas-control {
-    background: $header-color;
-    transform: scale(0.75);
-}
-
-.cvat-rotate-canvas-controls-left,
-.cvat-rotate-canvas-controls-right {
-    transform: scale(0.65);
-    border-radius: $grid-unit-size/2;
-
-    &:hover {
-        transform: scale(0.75);
-    }
-
-    &:active {
-        transform: scale(0.65);
-    }
-}
-
-.cvat-rotate-canvas-controls > .ant-popover-content > .ant-popover-inner > div > .ant-popover-inner-content {
-    padding: 0;
-}
-
-.cvat-draw-shape-popover,
-.cvat-tools-control-popover {
-    > .ant-popover-content > .ant-popover-inner > div > .ant-popover-inner-content {
-        padding: 0;
-    }
-}
-
-.cvat-tools-track-button,
-.cvat-tools-interact-button {
-    width: 100%;
-    margin-top: $grid-unit-size;
-}
-
-.cvat-draw-shape-popover-points-selector {
-    width: 100%;
-}
-
-.cvat-tools-control-popover-content {
-    width: fit-content;
-    padding: $grid-unit-size;
-    border-radius: $grid-unit-size/2;
-    background: $background-color-2;
-}
-
-.cvat-draw-shape-popover-content {
-    padding: $grid-unit-size;
-    border-radius: $grid-unit-size/2;
-    background: $background-color-2;
-    width: 270px;
-
-    > div {
-        margin-top: $grid-unit-size/2;
-    }
-
-    > div:nth-child(3) > div > div {
-        width: 100%;
-    }
-
-    > div:last-child {
-        span {
-            width: 100%;
-        }
-
-        button {
-            width: 100%;
-
-            &:nth-child(1) {
-                border-radius: $grid-unit-size/2 0 0 $grid-unit-size/2;
-            }
-
-            &:nth-child(2) {
-                border-radius: 0 $grid-unit-size/2 $grid-unit-size/2 0;
-            }
-        }
-    }
-}
-
 .cvat-canvas-container-overflow {
     overflow: hidden;
     width: 100%;
     height: 100%;
 }
 
-.cvat-control-side-bar-icon-size {
-    font-size: $grid-unit-size * 5;
-}
-
 .cvat-canvas3d-perspective {
     height: 100%;
     width: 100%;

cvat-ui/src/components/annotation-page/top-bar/top-bar.tsx

@@ -57,7 +57,6 @@ interface Props {
     onUndoClick(): void;
     onRedoClick(): void;
     jobInstance: any;
-    hideShowContextImage(): any;
 }
 
 export default function AnnotationTopBarComponent(props: Props): JSX.Element {

cvat-ui/src/containers/annotation-page/canvas/canvas-wrapper3D.tsx

@@ -5,7 +5,7 @@
 import { connect } from 'react-redux';
 
 import CanvasWrapperComponent from 'components/annotation-page/canvas/canvas-wrapper3D';
-import { confirmCanvasReady, getContextImage, resetCanvas } from 'actions/annotation-actions';
+import { confirmCanvasReady, resetCanvas } from 'actions/annotation-actions';
 
 import { CombinedState } from 'reducers/interfaces';
 
@@ -16,15 +16,11 @@ interface StateToProps {
     jobInstance: any;
     frameData: any;
     curZLayer: number;
-    contextImageHide: boolean;
-    loaded: boolean;
-    data: string;
     annotations: any[];
 }
 
 interface DispatchToProps {
     onSetupCanvas(): void;
-    getContextImage(): void;
     onResetCanvas(): void;
 }
 
@@ -35,7 +31,6 @@ function mapStateToProps(state: CombinedState): StateToProps {
             job: { instance: jobInstance },
             player: {
                 frame: { data: frameData },
-                contextImage: { hidden: contextImageHide, data, loaded },
             },
             annotations: {
                 states: annotations,
@@ -49,9 +44,6 @@ function mapStateToProps(state: CombinedState): StateToProps {
         jobInstance,
         frameData,
         curZLayer,
-        contextImageHide,
-        loaded,
-        data,
         annotations,
     };
 }
@@ -61,9 +53,6 @@ function mapDispatchToProps(dispatch: any): DispatchToProps {
         onSetupCanvas(): void {
             dispatch(confirmCanvasReady());
         },
-        getContextImage(): void {
-            dispatch(getContextImage());
-        },
         onResetCanvas(): void {
             dispatch(resetCanvas());
         },

cvat-ui/src/containers/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar.tsx

@@ -2,11 +2,10 @@
 //
 // SPDX-License-Identifier: MIT
 
-import { KeyMap } from 'utils/mousetrap-react';
 import { connect } from 'react-redux';
 
+import { KeyMap } from 'utils/mousetrap-react';
 import { Canvas } from 'cvat-canvas-wrapper';
-import { hideShowContextImage } from 'actions/annotation-actions';
 import ControlsSideBarComponent from 'components/annotation-page/standard3D-workspace/controls-side-bar/controls-side-bar';
 import { ActiveControl, CombinedState } from 'reducers/interfaces';
 
@@ -15,21 +14,13 @@ interface StateToProps {
     activeControl: ActiveControl;
     keyMap: KeyMap;
     normalizedKeyMap: Record<string, string>;
-    contextImageHide: boolean;
     loaded: boolean;
 }
 
-interface DispatchToProps {
-    hideShowContextImage(hidden: boolean): void;
-}
-
 function mapStateToProps(state: CombinedState): StateToProps {
     const {
         annotation: {
             canvas: { instance: canvasInstance, activeControl },
-            player: {
-                contextImage: { hidden: contextImageHide, loaded },
-            },
         },
         shortcuts: { keyMap, normalizedKeyMap },
     } = state;
@@ -39,17 +30,7 @@ function mapStateToProps(state: CombinedState): StateToProps {
         activeControl,
         normalizedKeyMap,
         keyMap,
-        contextImageHide,
-        loaded,
-    };
-}
-
-function dispatchToProps(dispatch: any): DispatchToProps {
-    return {
-        hideShowContextImage(hidden: boolean): void {
-            dispatch(hideShowContextImage(hidden));
-        },
     };
 }
 
-export default connect(mapStateToProps, dispatchToProps)(ControlsSideBarComponent);
+export default connect(mapStateToProps)(ControlsSideBarComponent);

cvat-ui/src/reducers/annotation-reducer.ts

@@ -51,6 +51,7 @@ const defaultState: AnnotationState = {
             number: 0,
             filename: '',
             data: null,
+            hasRelatedContext: false,
             fetching: false,
             delay: 0,
             changeTime: null,
@@ -58,8 +59,8 @@ const defaultState: AnnotationState = {
         playing: false,
         frameAngles: [],
         contextImage: {
-            loaded: false,
-            data: '',
+            fetching: false,
+            data: null,
             hidden: false,
         },
     },
@@ -145,6 +146,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                 openTime,
                 frameNumber: number,
                 frameFilename: filename,
+                frameHasRelatedContext,
                 colors,
                 filters,
                 frameData: data,
@@ -189,6 +191,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                     frame: {
                         ...state.player.frame,
                         filename,
+                        hasRelatedContext: frameHasRelatedContext,
                         number,
                         data,
                     },
@@ -226,11 +229,6 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                         ...state.player.frame,
                         fetching: false,
                     },
-                    contextImage: {
-                        loaded: false,
-                        data: '',
-                        hidden: state.player.contextImage.hidden,
-                    },
                 },
             };
         }
@@ -252,7 +250,16 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
         }
         case AnnotationActionTypes.CHANGE_FRAME_SUCCESS: {
             const {
-                number, data, filename, states, minZ, maxZ, curZ, delay, changeTime,
+                number,
+                data,
+                filename,
+                hasRelatedContext,
+                states,
+                minZ,
+                maxZ,
+                curZ,
+                delay,
+                changeTime,
             } = action.payload;
 
             const activatedStateID = states
@@ -268,6 +275,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                     frame: {
                         data,
                         filename,
+                        hasRelatedContext,
                         number,
                         fetching: false,
                         changeTime,
@@ -275,7 +283,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
                     },
                     contextImage: {
                         ...state.player.contextImage,
-                        loaded: false,
+                        data: null,
                     },
                 },
                 annotations: {
@@ -1170,30 +1178,52 @@ export default (state = defaultState, action: AnyAction): AnnotationState => {
         }
         case AnnotationActionTypes.HIDE_SHOW_CONTEXT_IMAGE: {
             const { hidden } = action.payload;
-            const { loaded, data } = state.player.contextImage;
             return {
                 ...state,
                 player: {
                     ...state.player,
                     contextImage: {
-                        loaded,
-                        data,
+                        ...state.player.contextImage,
                         hidden,
                     },
                 },
             };
         }
         case AnnotationActionTypes.GET_CONTEXT_IMAGE: {
-            const { context, loaded } = action.payload;
+            return {
+                ...state,
+                player: {
+                    ...state.player,
+                    contextImage: {
+                        ...state.player.contextImage,
+                        fetching: true,
+                    },
+                },
+            };
+        }
+        case AnnotationActionTypes.GET_CONTEXT_IMAGE_SUCCESS: {
+            const { contextImageData } = action.payload;
 
             return {
                 ...state,
                 player: {
                     ...state.player,
                     contextImage: {
-                        loaded,
-                        data: context,
-                        hidden: state.player.contextImage.hidden,
+                        ...state.player.contextImage,
+                        fetching: false,
+                        data: contextImageData,
+                    },
+                },
+            };
+        }
+        case AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED: {
+            return {
+                ...state,
+                player: {
+                    ...state.player,
+                    contextImage: {
+                        ...state.player.contextImage,
+                        fetching: false,
                     },
                 },
             };

cvat-ui/src/reducers/interfaces.ts

@@ -269,6 +269,7 @@ export interface NotificationsState {
             saving: null | ErrorState;
             jobFetching: null | ErrorState;
             frameFetching: null | ErrorState;
+            contextImageFetching: null | ErrorState;
             changingLabelColor: null | ErrorState;
             updating: null | ErrorState;
             creating: null | ErrorState;
@@ -417,6 +418,7 @@ export interface AnnotationState {
         frame: {
             number: number;
             filename: string;
+            hasRelatedContext: boolean;
             data: any | null;
             fetching: boolean;
             delay: number;
@@ -425,8 +427,8 @@ export interface AnnotationState {
         playing: boolean;
         frameAngles: number[];
         contextImage: {
-            loaded: boolean;
-            data: string;
+            fetching: boolean;
+            data: string | null;
             hidden: boolean;
         };
     };

cvat-ui/src/reducers/notifications-reducer.ts

@@ -69,6 +69,7 @@ const defaultState: NotificationsState = {
             saving: null,
             jobFetching: null,
             frameFetching: null,
+            contextImageFetching: null,
             changingLabelColor: null,
             updating: null,
             creating: null,
@@ -689,6 +690,21 @@ export default function (state = defaultState, action: AnyAction): Notifications
                 },
             };
         }
+        case AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED: {
+            return {
+                ...state,
+                errors: {
+                    ...state.errors,
+                    annotation: {
+                        ...state.errors.annotation,
+                        contextImageFetching: {
+                            message: 'Could not fetch context image from the server',
+                            reason: action.payload.error,
+                        },
+                    },
+                },
+            };
+        }
         case AnnotationActionTypes.SAVE_ANNOTATIONS_FAILED: {
             return {
                 ...state,

cvat/apps/engine/media_extractors.py

@@ -9,7 +9,6 @@ import zipfile
 import io
 import itertools
 import struct
-import re
 from abc import ABC, abstractmethod
 from contextlib import closing
 
@@ -112,6 +111,10 @@ class ImageListReader(IMediaReader):
         for i in range(self._start, self._stop, self._step):
             yield (self.get_image(i), self.get_path(i), i)
 
+    def filter(self, callback):
+        source_path = list(filter(callback, self._source_path))
+        ImageListReader.__init__(self, source_path, step=self._step, start=self._start, stop=self._stop)
+
     def get_path(self, i):
         return self._source_path[i]
 
@@ -199,7 +202,7 @@ class ZipReader(ImageListReader):
         self._zip_source = zipfile.ZipFile(source_path[0], mode='a')
         self.extract_dir = source_path[1] if len(source_path) > 1 else None
         file_list = [f for f in self._zip_source.namelist() if files_to_ignore(f) and get_mime(f) == 'image']
-        super().__init__(file_list, step, start, stop)
+        super().__init__(file_list, step=step, start=start, stop=stop)
 
     def __del__(self):
         self._zip_source.close()
@@ -759,66 +762,6 @@ class ValidateDimension:
                 self.image_files[file_name] = file_path
         return pcd_files
 
-    def validate_velodyne_points(self, *args):
-        root, actual_path, files = args
-        velodyne_files = self.process_files(root, actual_path, files)
-        related_path = os.path.split(os.path.split(root)[0])[0]
-
-        path_list = [re.search(r'image_\d.*', path, re.IGNORECASE) for path in os.listdir(related_path) if
-                     os.path.isdir(os.path.join(related_path, path))]
-
-        for path_ in path_list:
-            if path_:
-                path = os.path.join(path_.group(), "data")
-                path = os.path.abspath(os.path.join(related_path, path))
-
-                files = [file for file in os.listdir(path) if
-                         os.path.isfile(os.path.abspath(os.path.join(path, file)))]
-                for file in files:
-
-                    f_name = file.split(".")[0]
-                    if velodyne_files.get(f_name, None):
-                        self.related_files[velodyne_files[f_name]].append(
-                            os.path.abspath(os.path.join(path, file)))
-
-    def validate_pointcloud(self, *args):
-        root, actual_path, files = args
-        pointcloud_files = self.process_files(root, actual_path, files)
-        related_path = root.rsplit("/pointcloud", 1)[0]
-        related_images_path = os.path.join(related_path, "related_images")
-
-        if os.path.isdir(related_images_path):
-            paths = [path for path in os.listdir(related_images_path) if
-                     os.path.isdir(os.path.abspath(os.path.join(related_images_path, path)))]
-
-            for k in pointcloud_files:
-                for path in paths:
-
-                    if k == path.rsplit("_", 1)[0]:
-                        file_path = os.path.abspath(os.path.join(related_images_path, path))
-                        files = [file for file in os.listdir(file_path) if
-                                 os.path.isfile(os.path.join(file_path, file))]
-                        for related_image in files:
-                            self.related_files[pointcloud_files[k]].append(os.path.join(file_path, related_image))
-
-    def validate_default(self, *args):
-        root, actual_path, files = args
-        pcd_files = self.process_files(root, actual_path, files)
-        if len(list(pcd_files.keys())):
-
-            for image in self.image_files.keys():
-                if pcd_files.get(image, None):
-                    self.related_files[pcd_files[image]].append(self.image_files[image])
-
-            current_directory_name = os.path.split(root)
-
-            if len(pcd_files.keys()) == 1:
-                pcd_name = list(pcd_files.keys())[0].rsplit(".", 1)[0]
-                if current_directory_name[1] == pcd_name:
-                    for related_image in self.image_files.values():
-                        if root == os.path.split(related_image)[0]:
-                            self.related_files[pcd_files[pcd_name]].append(related_image)
-
     def validate(self):
         """
             Validate the directory structure for kitty and point cloud format.
@@ -830,15 +773,7 @@ class ValidateDimension:
             if not files_to_ignore(root):
                 continue
 
-            if root.endswith("data"):
-                if os.path.split(os.path.split(root)[0])[1] == "velodyne_points":
-                    self.validate_velodyne_points(root, actual_path, files)
-
-            elif os.path.split(root)[-1] == "pointcloud":
-                self.validate_pointcloud(root, actual_path, files)
-
-            else:
-                self.validate_default(root, actual_path, files)
+            self.process_files(root, actual_path, files)
 
         if len(self.related_files.keys()):
             self.dimension = DimensionType.DIM_3D

cvat/apps/engine/serializers.py

@@ -545,6 +545,7 @@ class FrameMetaSerializer(serializers.Serializer):
     width = serializers.IntegerField()
     height = serializers.IntegerField()
     name = serializers.CharField(max_length=1024)
+    has_related_context = serializers.BooleanField()
 
 class PluginsSerializer(serializers.Serializer):
     GIT_INTEGRATION = serializers.BooleanField()

cvat/apps/engine/task.py

@@ -7,6 +7,7 @@ import itertools
 import os
 import sys
 import rq
+import re
 import shutil
 from traceback import print_exception
 from urllib import parse as urlparse
@@ -19,6 +20,7 @@ from cvat.apps.engine.utils import av_scan_paths
 from cvat.apps.engine.models import DimensionType
 from utils.dataset_manifest import ImageManifestManager, VideoManifestManager
 from utils.dataset_manifest.core import VideoManifestValidator
+from utils.dataset_manifest.utils import detect_related_images
 
 import django_rq
 from django.conf import settings
@@ -273,10 +275,14 @@ def _create_thread(tid, data):
                 start=db_data.start_frame,
                 stop=data['stop_frame'],
                 dimension=DimensionType.DIM_3D,
-
             )
             extractor.add_files(validate_dimension.converted_files)
 
+    related_images = {}
+    if isinstance(extractor, MEDIA_TYPES['image']['extractor']):
+        extractor.filter(lambda x: not re.search(r'(^|{0})related_images{0}'.format(os.sep), x))
+        related_images = detect_related_images(extractor.absolute_source_paths, upload_dir)
+
     db_task.mode = task_mode
     db_data.compressed_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' and not data['use_zip_chunks'] else models.DataChoice.IMAGESET
     db_data.original_chunk_type = models.DataChoice.VIDEO if task_mode == 'interpolation' else models.DataChoice.IMAGESET
@@ -394,13 +400,14 @@ def _create_thread(tid, data):
                     base_msg = str(ex) if isinstance(ex, AssertionError) \
                         else "Uploaded video does not support a quick way of task creating."
                     _update_status("{} The task will be created using the old method".format(base_msg))
-            else:# images, archive, pdf
+            else: # images, archive, pdf
                 db_data.size = len(extractor)
                 manifest = ImageManifestManager(db_data.get_manifest_path())
                 if not manifest_file:
                     if db_task.dimension == DimensionType.DIM_2D:
                         meta_info = manifest.prepare_meta(
                             sources=extractor.absolute_source_paths,
+                            meta={ k: {'related_images': related_images[k] } for k in related_images },
                             data_dir=upload_dir
                         )
                         content = meta_info.content
@@ -410,6 +417,7 @@ def _create_thread(tid, data):
                             name, ext = os.path.splitext(os.path.relpath(source, upload_dir))
                             content.append({
                                 'name': name,
+                                'meta': { 'related_images': related_images[''.join((name, ext))] },
                                 'extension': ext
                             })
                     manifest.create(content)
@@ -465,27 +473,15 @@ def _create_thread(tid, data):
             update_progress(progress)
 
     if db_task.mode == 'annotation':
-        if validate_dimension.dimension == DimensionType.DIM_2D:
-            models.Image.objects.bulk_create(db_images)
-        else:
-            related_file = []
-            for image_data in db_images:
-                image_model = models.Image(
-                    data=image_data.data,
-                    path=image_data.path,
-                    frame=image_data.frame,
-                    width=image_data.width,
-                    height=image_data.height
-                )
-
-                image_model.save()
-                image_data = models.Image.objects.get(id=image_model.id)
-
-                if validate_dimension.related_files.get(image_data.path, None):
-                    for related_image_file in validate_dimension.related_files[image_data.path]:
-                        related_file.append(
-                            RelatedFile(data=db_data, primary_image_id=image_data.id, path=related_image_file))
-            RelatedFile.objects.bulk_create(related_file)
+        models.Image.objects.bulk_create(db_images)
+        created_images = models.Image.objects.filter(data_id=db_data.id)
+
+        db_related_files = [
+            RelatedFile(data=image.data, primary_image=image, path=os.path.join(upload_dir, related_file_path))
+            for image in created_images
+            for related_file_path in related_images.get(image.path, [])
+        ]
+        RelatedFile.objects.bulk_create(db_related_files)
         db_images = []
     else:
         models.Video.objects.create(

cvat/apps/engine/views.py

@@ -42,7 +42,7 @@ from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer
 from cvat.apps.engine.frame_provider import FrameProvider
 from cvat.apps.engine.models import (
     Job, StatusChoice, Task, Project, Review, Issue,
-    Comment, StorageMethodChoice, ReviewStatus, StorageChoice, DimensionType, Image
+    Comment, StorageMethodChoice, ReviewStatus, StorageChoice, Image
 )
 from cvat.apps.engine.serializers import (
     AboutSerializer, AnnotationFileSerializer, BasicUserSerializer,
@@ -487,21 +487,17 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
                     return sendfile(request, frame_provider.get_preview())
 
                 elif data_type == 'context_image':
-                    if db_task.dimension == DimensionType.DIM_3D:
-                        data_id = int(data_id)
-                        image = Image.objects.get(data_id=db_task.data_id, frame=data_id)
-                        for i in image.related_files.all():
-                            path = os.path.realpath(str(i.path))
-                            image = cv2.imread(path)
-                            success, result = cv2.imencode('.JPEG', image)
-                            if not success:
-                                raise Exception("Failed to encode image to '%s' format" % (".jpeg"))
-                            return HttpResponse(io.BytesIO(result.tobytes()), content_type="image/jpeg")
-                        return Response(data='No context image related to the frame',
-                                        status=status.HTTP_404_NOT_FOUND)
-                    else:
-                        return Response(data='Only 3D tasks support context images',
-                                        status=status.HTTP_400_BAD_REQUEST)
+                    data_id = int(data_id)
+                    image = Image.objects.get(data_id=db_data.id, frame=data_id)
+                    for i in image.related_files.all():
+                        path = os.path.realpath(str(i.path))
+                        image = cv2.imread(path)
+                        success, result = cv2.imencode('.JPEG', image)
+                        if not success:
+                            raise Exception('Failed to encode image to ".jpeg" format')
+                        return HttpResponse(io.BytesIO(result.tobytes()), content_type='image/jpeg')
+                    return Response(data='No context image related to the frame',
+                                    status=status.HTTP_404_NOT_FOUND)
                 else:
                     return Response(data='unknown data type {}.'.format(data_type), status=status.HTTP_400_BAD_REQUEST)
             except APIException as e:
@@ -636,7 +632,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
     @action(detail=True, methods=['GET'], serializer_class=DataMetaSerializer,
         url_path='data/meta')
     def data_info(request, pk):
-        db_task = models.Task.objects.prefetch_related('data__images').select_related('data__video').get(pk=pk)
+        db_task = models.Task.objects.prefetch_related('data__images__related_files').select_related('data__video').get(pk=pk)
 
         if hasattr(db_task.data, 'video'):
             media = [db_task.data.video]
@@ -647,6 +643,7 @@ class TaskViewSet(auth.TaskGetQuerySetMixin, viewsets.ModelViewSet):
             'width': item.width,
             'height': item.height,
             'name': item.path,
+            'has_related_context': hasattr(item, 'related_files') and bool(len(item.related_files.all()))
         } for item in media]
 
         db_data = db_task.data

tests/cypress/integration/canvas3d_functionality/case_56_canvas3d_functionality_basic_actions.js

@@ -57,10 +57,10 @@ context('Canvas 3D functionality. Basic actions.', () => {
     }
 
     function testContextImage() {
-        cy.get('.cvat-contextImage-show').should('exist').and('be.visible');
-        cy.get('[data-icon="camera"]').click(); // Context image hide
-        cy.get('.cvat-contextImage-show').should('not.exist');
-        cy.get('[data-icon="camera"]').click(); // Context image show
+        cy.get('.cvat-context-image-wrapper img').should('exist').and('be.visible');
+        cy.get('.cvat-context-image-switcher').click(); // Context image hide
+        cy.get('.cvat-context-image-wrapper img').should('not.exist');
+        cy.get('.cvat-context-image-switcher').click(); // Context image show
     }
 
     function testControlButtonTooltip(button, expectedTooltipText) {
@@ -104,9 +104,11 @@ context('Canvas 3D functionality. Basic actions.', () => {
             cy.get('.cvat-canvas3d-topview').should('exist').and('be.visible');
             cy.get('.cvat-canvas3d-sideview').should('exist').and('be.visible');
             cy.get('.cvat-canvas3d-frontview').should('exist').and('be.visible');
-            cy.get('.cvat-canvas-controls-sidebar').find('[role="img"]').then(($controlButtons) => {
-                expect($controlButtons.length).to.be.equal(4);
-            });
+            cy.get('.cvat-canvas-controls-sidebar')
+                .find('[role="img"]')
+                .then(($controlButtons) => {
+                    expect($controlButtons.length).to.be.equal(3);
+                });
             cy.get('.cvat-canvas-controls-sidebar')
                 .should('exist')
                 .and('be.visible')
@@ -114,12 +116,10 @@ context('Canvas 3D functionality. Basic actions.', () => {
                     cy.get('.cvat-move-control').should('exist').and('be.visible');
                     cy.get('.cvat-cursor-control').should('exist').and('be.visible');
                     cy.get('.cvat-draw-cuboid-control').should('exist').and('be.visible');
-                    cy.get('.cvat-context-image-control').should('exist').and('be.visible');
                 });
             [
                 ['.cvat-move-control', 'Move the image'],
                 ['.cvat-cursor-control', 'Cursor [Esc]'],
-                ['.cvat-context-image-control', 'Photo context show/hide']
             ].forEach(([button, tooltip]) => {
                 testControlButtonTooltip(button, tooltip);
             });

tests/cypress/integration/canvas3d_functionality/case_63_canvas3d_functionality_control_button_mouse_interaction.js

@@ -34,7 +34,6 @@ context('Canvas 3D functionality. Control button. Mouse interaction.', () => {
 
     before(() => {
         cy.openTaskJob(taskName);
-        cy.get('.cvat-contextImage-show').should('be.visible');
     });
 
     describe(`Testing case "${caseId}"`, () => {

utils/dataset_manifest/core.py

@@ -145,8 +145,9 @@ class VideoStreamReader:
 
 
 class DatasetImagesReader:
-    def __init__(self, sources, is_sorted=True, use_image_hash=False, *args, **kwargs):
+    def __init__(self, sources, meta=None, is_sorted=True, use_image_hash=False, *args, **kwargs):
         self._sources = sources if is_sorted else sorted(sources)
+        self._meta = meta
         self._content = []
         self._data_dir = kwargs.get('data_dir', None)
         self._use_image_hash = use_image_hash
@@ -163,6 +164,8 @@ class DatasetImagesReader:
                 'width': img.width,
                 'height': img.height,
             }
+            if self._meta and img_name in self._meta:
+                image_properties['meta'] = self._meta[img_name]
             if self._use_image_hash:
                 image_properties['checksum'] = md5_hash(img)
             yield image_properties
@@ -177,7 +180,7 @@ class DatasetImagesReader:
 
 class _Manifest:
     FILE_NAME = 'manifest.jsonl'
-    VERSION = '1.0'
+    VERSION = '1.1'
 
     def __init__(self, path, is_created=False):
         assert path, 'A path to manifest file not found'

utils/dataset_manifest/create.py

@@ -2,21 +2,12 @@
 #
 # SPDX-License-Identifier: MIT
 import argparse
-import mimetypes
 import os
 import sys
+import re
 from glob import glob
 
-def _define_data_type(media):
-    media_type, _ = mimetypes.guess_type(media)
-    if media_type:
-        return media_type.split('/')[0]
-
-def _is_video(media_file):
-    return _define_data_type(media_file) == 'video'
-
-def _is_image(media_file):
-    return _define_data_type(media_file) == 'image'
+from utils import detect_related_images, is_image, is_video
 
 def get_args():
     parser = argparse.ArgumentParser()
@@ -33,7 +24,7 @@ def main():
 
     manifest_directory = os.path.abspath(args.output_dir)
     os.makedirs(manifest_directory, exist_ok=True)
-    source = os.path.abspath(args.source)
+    source = os.path.abspath(os.path.expanduser(args.source))
 
     sources = []
     if not os.path.isfile(source): # directory/pattern with images
@@ -41,7 +32,7 @@ def main():
         if os.path.isdir(source):
             data_dir = source
             for root, _, files in os.walk(source):
-                sources.extend([os.path.join(root, f) for f in files if _is_image(f)])
+                sources.extend([os.path.join(root, f) for f in files if is_image(f)])
         else:
             items = source.lstrip('/').split('/')
             position = 0
@@ -56,18 +47,28 @@ def main():
                 data_dir = source.split(items[position])[0]
             except Exception as ex:
                 sys.exit(str(ex))
-            sources = list(filter(_is_image, glob(source, recursive=True)))
+            sources = list(filter(is_image, glob(source, recursive=True)))
+
+        sources = list(filter(lambda x: 'related_images{}'.format(os.sep) not in x, sources))
+
+        # If the source is a glob expression, we need additional processing
+        abs_root = source
+        while abs_root and re.search('[*?\[\]]', abs_root):
+            abs_root = os.path.split(abs_root)[0]
+
+        related_images = detect_related_images(sources, abs_root)
+        meta = { k: {'related_images': related_images[k] } for k in related_images }
         try:
             assert len(sources), 'A images was not found'
             manifest = ImageManifestManager(manifest_path=manifest_directory)
-            meta_info = manifest.prepare_meta(sources=sources, is_sorted=False,
+            meta_info = manifest.prepare_meta(sources=sources, meta=meta, is_sorted=False,
                 use_image_hash=True, data_dir=data_dir)
             manifest.create(meta_info)
         except Exception as ex:
             sys.exit(str(ex))
     else: # video
         try:
-            assert _is_video(source), 'You can specify a video path or a directory/pattern with images'
+            assert is_video(source), 'You can specify a video path or a directory/pattern with images'
             manifest = VideoManifestManager(manifest_path=manifest_directory)
             try:
                 meta_info = manifest.prepare_meta(media_file=source, force=args.force)

utils/dataset_manifest/utils.py

 # Copyright (C) 2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
+import os
+import re
 import hashlib
+import mimetypes
 import cv2 as cv
 from av import VideoFrame
 
@@ -21,4 +24,163 @@ def rotate_image(image, angle):
 def md5_hash(frame):
     if isinstance(frame, VideoFrame):
         frame = frame.to_image()
-    return hashlib.md5(frame.tobytes()).hexdigest() # nosec
\ No newline at end of file
+    return hashlib.md5(frame.tobytes()).hexdigest() # nosec
+
+def _define_data_type(media):
+    return mimetypes.guess_type(media)[0]
+
+def is_video(media_file):
+    data_type = _define_data_type(media_file)
+    return data_type is not None and data_type.startswith('video')
+
+def is_image(media_file):
+    data_type = _define_data_type(media_file)
+    return data_type is not None and data_type.startswith('image') and \
+        not data_type.startswith('image/svg')
+
+
+def _list_and_join(root):
+    files = os.listdir(root)
+    for f in files:
+        yield os.path.join(root, f)
+
+def _prepare_context_list(files, base_dir):
+    return sorted(map(lambda x: os.path.relpath(x, base_dir), filter(is_image, files)))
+
+# Expected 2D format is:
+# data/
+#   00001.png
+#   related_images/
+#     00001_png/
+#       context_image_1.jpeg
+#       context_image_2.png
+def _detect_related_images_2D(image_paths, root_path):
+    related_images = {}
+    latest_dirname = ''
+    related_images_exist = False
+
+    for image_path in sorted(image_paths):
+        rel_image_path = os.path.relpath(image_path, root_path)
+        dirname = os.path.dirname(image_path)
+        related_images_dirname = os.path.join(dirname, 'related_images')
+        related_images[rel_image_path] = []
+
+        if latest_dirname == dirname and not related_images_exist:
+            continue
+        elif latest_dirname != dirname:
+            # Update some data applicable for a subset of paths (within the current dirname)
+            latest_dirname = dirname
+            related_images_exist = os.path.isdir(related_images_dirname)
+
+        if related_images_exist:
+            related_images_dirname = os.path.join(
+                related_images_dirname, '_'.join(os.path.basename(image_path).rsplit('.', 1))
+            )
+
+            if os.path.isdir(related_images_dirname):
+                related_images[rel_image_path] = _prepare_context_list(_list_and_join(related_images_dirname), root_path)
+    return related_images
+
+# Possible 3D formats are:
+# velodyne_points/
+#     data/
+#         image_01.bin
+# IMAGE_00 # any number?
+#     data/
+#         image_01.png
+
+# pointcloud/
+#     00001.pcd
+# related_images/
+#     00001_pcd/
+#         image_01.png # or other image
+
+# Default formats
+# Option 1
+# data/
+#     image.pcd
+#     image.png
+
+# Option 2
+# data/
+#    image_1/
+#        image_1.pcd
+#        context_1.png
+#        context_2.jpg
+def _detect_related_images_3D(image_paths, root_path):
+    related_images = {}
+    latest_dirname = ''
+    dirname_files = []
+    related_images_exist = False
+    velodyne_context_images_dirs = []
+
+    for image_path in sorted(image_paths):
+        rel_image_path = os.path.relpath(image_path, root_path)
+        name = os.path.splitext(os.path.basename(image_path))[0]
+        dirname = os.path.dirname(image_path)
+        related_images_dirname = os.path.normpath(os.path.join(dirname, '..', 'related_images'))
+        related_images[rel_image_path] = []
+
+        if latest_dirname != dirname:
+            # Update some data applicable for a subset of paths (within the current dirname)
+            latest_dirname = dirname
+            related_images_exist = os.path.isdir(related_images_dirname)
+            dirname_files = list(filter(lambda x: x != image_path, _list_and_join(dirname)))
+            velodyne_context_images_dirs = [directory for directory
+                in _list_and_join(os.path.normpath(os.path.join(dirname, '..', '..')))
+                if os.path.isdir(os.path.join(directory, 'data')) and re.search(r'image_\d.*', directory, re.IGNORECASE)
+            ]
+
+        if os.path.basename(dirname) == name:
+            # default format (option 2)
+            related_images[rel_image_path].extend(_prepare_context_list(dirname_files, root_path))
+
+        filtered_dirname_files = list(filter(lambda x: os.path.splitext(os.path.basename(x))[0] == name, dirname_files))
+        if len(filtered_dirname_files):
+            # default format (option 1)
+            related_images[rel_image_path].extend(_prepare_context_list(filtered_dirname_files, root_path))
+
+        if related_images_exist:
+            related_images_dirname = os.path.join(
+                related_images_dirname, '_'.join(os.path.basename(image_path).rsplit('.', 1))
+            )
+            if os.path.isdir(related_images_dirname):
+                related_images[rel_image_path].extend(
+                    _prepare_context_list(_list_and_join(related_images_dirname), root_path)
+                )
+
+        if dirname.endswith(os.path.join('velodyne_points', 'data')):
+            # velodynepoints format
+            for context_images_dir in velodyne_context_images_dirs:
+                context_files = _list_and_join(os.path.join(context_images_dir, 'data'))
+                context_files = list(
+                    filter(lambda x: os.path.splitext(os.path.basename(x))[0] == name, context_files)
+                )
+                related_images[rel_image_path].extend(
+                    _prepare_context_list(context_files, root_path)
+                )
+
+        related_images[rel_image_path].sort()
+    return related_images
+
+# This function is expected to be called only for images tasks
+# image_path is expected to be a list of absolute path to images
+# root_path is expected to be a string (dataset root)
+def detect_related_images(image_paths, root_path):
+    data_are_2d = False
+    data_are_3d = False
+
+    # First of all need to define data type we are working with
+    for image_path in image_paths:
+        # .bin files are expected to be converted to .pcd before this code
+        if os.path.splitext(image_path)[1].lower() == '.pcd':
+            data_are_3d = True
+        else:
+            data_are_2d = True
+    assert not (data_are_3d and data_are_2d), 'Combined data types 2D and 3D are not supported'
+
+    if data_are_2d:
+        return _detect_related_images_2D(image_paths, root_path)
+    elif data_are_3d:
+        return _detect_related_images_3D(image_paths, root_path)
+    return {}