[cvat-core] support cloud storage (#3313)

* Update server-proxy

* Add cloud storage implementation && update enums

* Add api && fix server-proxy

* Add fixes

* small update

* Move from ui_support_cloud_storage

* Remove temporary credentials && fix typos

* Remove trailing spaces

* Apply lost changes

* manifest_set -> manifests

* [cvat-core] Add status && updated getpreview for getting the desired error message from the server

* Remove excess code

* Fix missing

* Add tests

* move from cycle

* Update CHANGELOG

* Increase version

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added intelligent scissors blocking feature (<https://github.com/openvinotoolkit/cvat/pull/3510>)
 - Support cloud storage status (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 - Support cloud storage preview (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- cvat-core: support cloud storages (<https://github.com/openvinotoolkit/cvat/pull/3313>)
 
 ### Changed
 

cvat-core/package-lock.json

 {
   "name": "cvat-core",
-  "version": "3.15.0",
+  "version": "3.16.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

cvat-core/package.json

 {
   "name": "cvat-core",
-  "version": "3.15.0",
+  "version": "3.16.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/api-implementation.js

@@ -16,13 +16,20 @@
         camelToSnake,
     } = require('./common');
 
-    const { TaskStatus, TaskMode, DimensionType } = require('./enums');
+    const {
+        TaskStatus,
+        TaskMode,
+        DimensionType,
+        CloudStorageProviderType,
+        CloudStorageCredentialsType,
+    } = require('./enums');
 
     const User = require('./user');
     const { AnnotationFormats } = require('./annotation-formats');
     const { ArgumentError } = require('./exceptions');
     const { Task } = require('./session');
     const { Project } = require('./project');
+    const { CloudStorage } = require('./cloud-storage');
 
     function implementAPI(cvat) {
         cvat.plugins.list.implementation = PluginRegistry.list;
@@ -262,6 +269,49 @@
 
         cvat.projects.searchNames.implementation = async (search, limit) => serverProxy.projects.searchNames(search, limit);
 
+        cvat.cloudStorages.get.implementation = async (filter) => {
+            checkFilter(filter, {
+                page: isInteger,
+                displayName: isString,
+                resourceName: isString,
+                description: isString,
+                id: isInteger,
+                owner: isString,
+                search: isString,
+                providerType: isEnum.bind(CloudStorageProviderType),
+                credentialsType: isEnum.bind(CloudStorageCredentialsType),
+            });
+
+            checkExclusiveFields(filter, ['id', 'search'], ['page']);
+
+            const searchParams = new URLSearchParams();
+            for (const field of [
+                'displayName',
+                'credentialsType',
+                'providerType',
+                'owner',
+                'search',
+                'id',
+                'page',
+                'description',
+            ]) {
+                if (Object.prototype.hasOwnProperty.call(filter, field)) {
+                    searchParams.set(camelToSnake(field), filter[field]);
+                }
+            }
+
+            if (Object.prototype.hasOwnProperty.call(filter, 'resourceName')) {
+                searchParams.set('resource', filter.resourceName);
+            }
+
+            const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams.toString());
+            const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage));
+
+            cloudStorages.count = cloudStoragesData.count;
+
+            return cloudStorages;
+        };
+
         return cvat;
     }
 

cvat-core/src/api.js

@@ -22,6 +22,8 @@ function build() {
     const { Attribute, Label } = require('./labels');
     const MLModel = require('./ml-model');
     const { FrameData } = require('./frames');
+    const { CloudStorage } = require('./cloud-storage');
+
 
     const enums = require('./enums');
 
@@ -748,6 +750,41 @@ function build() {
             PluginError,
             ServerError,
         },
+        /**
+         * Namespace is used for getting cloud storages
+         * @namespace cloudStorages
+         * @memberof module:API.cvat
+         */
+        cloudStorages: {
+            /**
+             * @typedef {Object} CloudStorageFilter
+             * @property {string} displayName Check if displayName contains this value
+             * @property {string} resourceName Check if resourceName contains this value
+             * @property {module:API.cvat.enums.ProviderType} providerType Check if providerType equal this value
+             * @property {integer} id Check if id equals this value
+             * @property {integer} page Get specific page
+             * (default REST API returns 20 clouds storages per request.
+             * In order to get more, it is need to specify next page)
+             * @property {string} owner Check if an owner name contains this value
+             * @property {string} search Combined search of contains among all the fields
+             * @global
+             */
+
+            /**
+             * Method returns a list of cloud storages corresponding to a filter
+             * @method get
+             * @async
+             * @memberof module:API.cvat.cloudStorages
+             * @param {CloudStorageFilter} [filter={}] cloud storage filter
+             * @returns {module:API.cvat.classes.CloudStorage[]}
+             * @throws {module:API.cvat.exceptions.PluginError}
+             * @throws {module:API.cvat.exceptions.ServerError}
+             */
+            async get(filter = {}) {
+                const result = await PluginRegistry.apiWrapper(cvat.cloudStorages.get, filter);
+                return result;
+            },
+        },
         /**
          * Namespace is used for access to classes
          * @namespace classes
@@ -768,6 +805,7 @@ function build() {
             Issue,
             Review,
             FrameData,
+            CloudStorage,
         },
     };
 
@@ -780,6 +818,7 @@ function build() {
     cvat.lambda = Object.freeze(cvat.lambda);
     cvat.client = Object.freeze(cvat.client);
     cvat.enums = Object.freeze(cvat.enums);
+    cvat.cloudStorages = Object.freeze(cvat.cloudStorages);
 
     const implementAPI = require('./api-implementation');
 

cvat-core/src/enums.js

-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -333,6 +333,36 @@
         '#733380',
     ];
 
+    /**
+     * Types of cloud storage providers
+     * @enum {string}
+     * @name CloudStorageProviderType
+     * @memberof module:API.cvat.enums
+     * @property {string} AWS_S3 'AWS_S3_BUCKET'
+     * @property {string} AZURE 'AZURE_CONTAINER'
+     * @readonly
+     */
+    const CloudStorageProviderType = Object.freeze({
+        AWS_S3_BUCKET: 'AWS_S3_BUCKET',
+        AZURE_CONTAINER: 'AZURE_CONTAINER',
+    });
+
+    /**
+     * Types of cloud storage credentials
+     * @enum {string}
+     * @name CloudStorageCredentialsType
+     * @memberof module:API.cvat.enums
+     * @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR'
+     * @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR'
+     * @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS'
+     * @readonly
+     */
+    const CloudStorageCredentialsType = Object.freeze({
+        KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR',
+        ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR',
+        ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS',
+    });
+
     module.exports = {
         ShareFileType,
         TaskStatus,
@@ -348,5 +378,7 @@
         colors,
         Source,
         DimensionType,
+        CloudStorageProviderType,
+        CloudStorageCredentialsType,
     };
 })();

cvat-core/src/server-proxy.js

@@ -1145,9 +1145,7 @@
 
                     const closureId = Date.now();
                     predictAnnotations.latestRequest.id = closureId;
-                    const predicate = () => (
-                        !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId
-                    );
+                    const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId;
                     if (predictAnnotations.latestRequest.fetching) {
                         waitFor(5, predicate).then(() => {
                             if (predictAnnotations.latestRequest.id !== closureId) {
@@ -1181,6 +1179,121 @@
                 }
             }
 
+            async function createCloudStorage(storageDetail) {
+                const { backendAPI } = config;
+
+                try {
+                    const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), {
+                        proxy: config.proxy,
+                        headers: {
+                            'Content-Type': 'application/json',
+                        },
+                    });
+                    return response.data;
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+            }
+
+            async function updateCloudStorage(id, cloudStorageData) {
+                const { backendAPI } = config;
+
+                try {
+                    await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), {
+                        proxy: config.proxy,
+                        headers: {
+                            'Content-Type': 'application/json',
+                        },
+                    });
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+            }
+
+            async function getCloudStorages(filter = '') {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    response = await Axios.get(`${backendAPI}/cloudstorages?page_size=12&${filter}`, {
+                        proxy: config.proxy,
+                    });
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+
+                response.data.results.count = response.data.count;
+                return response.data.results;
+            }
+
+            async function getCloudStorageContent(id, manifestPath) {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    const url = `${backendAPI}/cloudstorages/${id}/content${
+                        manifestPath ? `?manifest_path=${manifestPath}` : ''
+                    }`;
+                    response = await Axios.get(url, {
+                        proxy: config.proxy,
+                    });
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+
+                return response.data;
+            }
+
+            async function getCloudStoragePreview(id) {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    const url = `${backendAPI}/cloudstorages/${id}/preview`;
+                    response = await workerAxios.get(url, {
+                        proxy: config.proxy,
+                        responseType: 'arraybuffer',
+                    });
+                } catch (errorData) {
+                    throw generateError({
+                        ...errorData,
+                        message: '',
+                        response: {
+                            ...errorData.response,
+                            data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)),
+                        },
+                    });
+                }
+
+                return new Blob([new Uint8Array(response)]);
+            }
+
+            async function getCloudStorageStatus(id) {
+                const { backendAPI } = config;
+
+                let response = null;
+                try {
+                    const url = `${backendAPI}/cloudstorages/${id}/status`;
+                    response = await Axios.get(url, {
+                        proxy: config.proxy,
+                    });
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+
+                return response.data;
+            }
+
+            async function deleteCloudStorage(id) {
+                const { backendAPI } = config;
+
+                try {
+                    await Axios.delete(`${backendAPI}/cloudstorages/${id}`);
+                } catch (errorData) {
+                    throw generateError(errorData);
+                }
+            }
+
             Object.defineProperties(
                 this,
                 Object.freeze({
@@ -1310,6 +1423,19 @@
                         }),
                         writable: false,
                     },
+
+                    cloudStorages: {
+                        value: Object.freeze({
+                            get: getCloudStorages,
+                            getContent: getCloudStorageContent,
+                            getPreview: getCloudStoragePreview,
+                            getStatus: getCloudStorageStatus,
+                            create: createCloudStorage,
+                            delete: deleteCloudStorage,
+                            update: updateCloudStorage,
+                        }),
+                        writable: false,
+                    },
                 }),
             );
         }

cvat-core/src/session.js

@@ -1012,6 +1012,7 @@
                 use_cache: undefined,
                 copy_data: undefined,
                 dimension: undefined,
+                cloud_storage_id: undefined,
             };
 
             const updatedFields = new FieldUpdateTrigger({
@@ -1373,7 +1374,7 @@
                         get: () => [...data.jobs],
                     },
                     /**
-                     * List of files from shared resource
+                     * List of files from shared resource or list of cloud storage files
                      * @name serverFiles
                      * @type {string[]}
                      * @memberof module:API.cvat.classes.Task
@@ -1535,6 +1536,15 @@
                          */
                         get: () => data.dimension,
                     },
+                    /**
+                     * @name cloudStorageId
+                     * @type {integer|null}
+                     * @memberof module:API.cvat.classes.Task
+                     * @instance
+                     */
+                    cloudStorageId: {
+                        get: () => data.cloud_storage_id,
+                    },
                     _internalData: {
                         get: () => data,
                     },
@@ -2062,6 +2072,9 @@
         if (typeof this.copyData !== 'undefined') {
             taskDataSpec.copy_data = this.copyData;
         }
+        if (typeof this.cloudStorageId !== 'undefined') {
+            taskDataSpec.cloud_storage_id = this.cloudStorageId;
+        }
 
         const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate);
         return new Task(task);

cvat-core/tests/api/cloud-storages.js

+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+// Setup mock for a server
+jest.mock('../../src/server-proxy', () => {
+    const mock = require('../mocks/server-proxy.mock');
+    return mock;
+});
+
+// Initialize api
+window.cvat = require('../../src/api');
+
+const { CloudStorage } = require('../../src/cloud-storage');
+const { cloudStoragesDummyData } = require('../mocks/dummy-data.mock');
+
+describe('Feature: get cloud storages', () => {
+    test('get all cloud storages', async () => {
+        const result = await window.cvat.cloudStorages.get();
+        expect(Array.isArray(result)).toBeTruthy();
+        expect(result).toHaveLength(cloudStoragesDummyData.count);
+        for (const item of result) {
+            expect(item).toBeInstanceOf(CloudStorage);
+        }
+    });
+
+    test('get cloud storage by id', async () => {
+        const result = await window.cvat.cloudStorages.get({
+            id: 1,
+        });
+        const cloudStorage = result[0];
+
+        expect(Array.isArray(result)).toBeTruthy();
+        expect(result).toHaveLength(1);
+        expect(cloudStorage).toBeInstanceOf(CloudStorage);
+        expect(cloudStorage.id).toBe(1);
+        expect(cloudStorage.providerType).toBe('AWS_S3_BUCKET');
+        expect(cloudStorage.credentialsType).toBe('KEY_SECRET_KEY_PAIR');
+        expect(cloudStorage.resourceName).toBe('bucket');
+        expect(cloudStorage.displayName).toBe('Demonstration bucket');
+        expect(cloudStorage.manifests).toHaveLength(1);
+        expect(cloudStorage.manifests[0]).toBe('manifest.jsonl');
+        expect(cloudStorage.specificAttributes).toBe('');
+        expect(cloudStorage.description).toBe('It is first bucket');
+    });
+
+    test('get a cloud storage by an unknown id', async () => {
+        const result = await window.cvat.cloudStorages.get({
+            id: 10,
+        });
+        expect(Array.isArray(result)).toBeTruthy();
+        expect(result).toHaveLength(0);
+    });
+
+    test('get a cloud storage by an invalid id', async () => {
+        expect(
+            window.cvat.cloudStorages.get({
+                id: '1',
+            }),
+        ).rejects.toThrow(window.cvat.exceptions.ArgumentError);
+    });
+
+    test('get cloud storages by filters', async () => {
+        const filters = new Map([
+            ['providerType', 'AWS_S3_BUCKET'],
+            ['resourceName', 'bucket'],
+            ['displayName', 'Demonstration bucket'],
+            ['credentialsType', 'KEY_SECRET_KEY_PAIR'],
+            ['description', 'It is first bucket'],
+        ]);
+
+        const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters));
+
+        const [cloudStorage] = result;
+        expect(Array.isArray(result)).toBeTruthy();
+        expect(result).toHaveLength(1);
+        expect(cloudStorage).toBeInstanceOf(CloudStorage);
+        expect(cloudStorage.id).toBe(1);
+        filters.forEach((value, key) => {
+            expect(cloudStorage[key]).toBe(value);
+        });
+    });
+
+    test('get cloud storage by invalid filters', async () => {
+        expect(
+            window.cvat.cloudStorages.get({
+                unknown: '5',
+            }),
+        ).rejects.toThrow(window.cvat.exceptions.ArgumentError);
+    });
+});
+
+describe('Feature: create a cloud storage', () => {
+    test('create new cloud storage without an id', async () => {
+        const cloudStorage = new window.cvat.classes.CloudStorage({
+            display_name: 'new cloud storage',
+            provider_type: 'AZURE_CONTAINER',
+            resource: 'newcontainer',
+            credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR',
+            account_name: 'accountname',
+            session_token: 'x'.repeat(135),
+            manifests: ['manifest.jsonl'],
+        });
+
+        const result = await cloudStorage.save();
+        expect(typeof result.id).toBe('number');
+    });
+});
+
+describe('Feature: update a cloud storage', () => {
+    test('update cloud storage with some new field values', async () => {
+        const newValues = new Map([
+            ['displayName', 'new display name'],
+            ['credentialsType', 'ANONYMOUS_ACCESS'],
+            ['description', 'new description'],
+            ['specificAttributes', 'region=eu-west-1'],
+        ]);
+
+        let result = await window.cvat.cloudStorages.get({
+            id: 1,
+        });
+
+        let [cloudStorage] = result;
+
+        for (const [key, value] of newValues) {
+            cloudStorage[key] = value;
+        }
+
+        cloudStorage.save();
+
+        result = await window.cvat.cloudStorages.get({
+            id: 1,
+        });
+        [cloudStorage] = result;
+
+        newValues.forEach((value, key) => {
+            expect(cloudStorage[key]).toBe(value);
+        });
+    });
+
+    test('Update manifests in a cloud storage', async () => {
+        const newManifests = [
+            'sub1/manifest.jsonl',
+            'sub2/manifest.jsonl',
+        ];
+
+        let result = await window.cvat.cloudStorages.get({
+            id: 1,
+        });
+        let [cloudStorage] = result;
+
+        cloudStorage.manifests = newManifests;
+        cloudStorage.save();
+
+        result = await window.cvat.cloudStorages.get({
+            id: 1,
+        });
+        [cloudStorage] = result;
+
+        expect(cloudStorage.manifests).toEqual(newManifests);
+    });
+});
+
+describe('Feature: delete a cloud storage', () => {
+    test('delete a cloud storage', async () => {
+        let result = await window.cvat.cloudStorages.get({
+            id: 2,
+        });
+
+        await result[0].delete();
+        result = await window.cvat.cloudStorages.get({
+            id: 2,
+        });
+
+        expect(Array.isArray(result)).toBeTruthy();
+        expect(result).toHaveLength(0);
+    });
+});

cvat-core/tests/mocks/dummy-data.mock.js

@@ -2547,6 +2547,56 @@ const frameMetaDummyData = {
     },
 };
 
+const cloudStoragesDummyData = {
+    count: 2,
+    next: null,
+    previous: null,
+    results: [
+        {
+            id: 2,
+            owner: {
+                url: 'http://localhost:7000/api/v1/users/1',
+                id: 1,
+                username: 'maya',
+                first_name: '',
+                last_name: ''
+            },
+            manifests: [
+                'manifest.jsonl'
+            ],
+            provider_type: 'AZURE_CONTAINER',
+            resource: 'container',
+            display_name: 'Demonstration container',
+            created_date: '2021-09-01T09:29:47.094244Z',
+            updated_date: '2021-09-01T09:29:47.103264Z',
+            credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR',
+            specific_attributes: '',
+            description: 'It is first container'
+        },
+        {
+            id: 1,
+            owner: {
+                url: 'http://localhost:7000/api/v1/users/1',
+                id: 1,
+                username: 'maya',
+                first_name: '',
+                last_name: ''
+            },
+            manifests: [
+                'manifest.jsonl'
+            ],
+            provider_type: 'AWS_S3_BUCKET',
+            resource: 'bucket',
+            display_name: 'Demonstration bucket',
+            created_date: '2021-08-31T09:03:09.350817Z',
+            updated_date: '2021-08-31T15:16:21.394773Z',
+            credentials_type: 'KEY_SECRET_KEY_PAIR',
+            specific_attributes: '',
+            description: 'It is first bucket'
+        }
+    ]
+};
+
 module.exports = {
     tasksDummyData,
     projectsDummyData,
@@ -2557,4 +2607,5 @@ module.exports = {
     jobAnnotationsDummyData,
     frameMetaDummyData,
     formatsDummyData,
+    cloudStoragesDummyData,
 };

cvat-core/tests/mocks/server-proxy.mock.js

@@ -12,6 +12,7 @@ const {
     taskAnnotationsDummyData,
     jobAnnotationsDummyData,
     frameMetaDummyData,
+    cloudStoragesDummyData,
 } = require('./dummy-data.mock');
 
 function QueryStringToJSON(query, ignoreList = []) {
@@ -318,6 +319,63 @@ class ServerProxy {
             return null;
         }
 
+        async function getCloudStorages(filter = '') {
+            const queries = QueryStringToJSON(filter);
+            const result = cloudStoragesDummyData.results.filter((item) => {
+                for (const key in queries) {
+                    if (Object.prototype.hasOwnProperty.call(queries, key)) {
+                        if (queries[key] !== item[key]) {
+                            return false;
+                        }
+                    }
+                }
+                return true;
+            });
+            return result;
+        }
+
+        async function updateCloudStorage(id, cloudStorageData) {
+            const cloudStorage = cloudStoragesDummyData.results.find((item) => item.id === id);
+            if (cloudStorage) {
+                for (const prop in cloudStorageData) {
+                    if (
+                        Object.prototype.hasOwnProperty.call(cloudStorageData, prop)
+                            && Object.prototype.hasOwnProperty.call(cloudStorage, prop)
+                    ) {
+                        cloudStorage[prop] = cloudStorageData[prop];
+                    }
+                }
+            }
+        }
+
+        async function createCloudStorage(cloudStorageData) {
+            const id = Math.max(...cloudStoragesDummyData.results.map((item) => item.id)) + 1;
+            cloudStoragesDummyData.results.push({
+                id,
+                provider_type: cloudStorageData.provider_type,
+                resource: cloudStorageData.resource,
+                display_name: cloudStorageData.display_name,
+                credentials_type: cloudStorageData.credentials_type,
+                specific_attributes: cloudStorageData.specific_attributes,
+                description: cloudStorageData.description,
+                owner: 1,
+                created_date: '2021-09-01T09:29:47.094244+03:00',
+                updated_date: '2021-09-01T09:29:47.103264+03:00',
+            });
+
+            const result = await getCloudStorages(`?id=${id}`);
+            return result[0];
+        }
+
+        async function deleteCloudStorage(id) {
+            const cloudStorages = cloudStoragesDummyData.results;
+            const cloudStorageId = cloudStorages.findIndex((item) => item.id === id);
+            if (cloudStorageId !== -1) {
+                cloudStorages.splice(cloudStorageId);
+            }
+        }
+
+
         Object.defineProperties(
             this,
             Object.freeze({
@@ -384,6 +442,16 @@ class ServerProxy {
                         getAnnotations,
                     },
                 },
+
+                cloudStorages: {
+                    value: Object.freeze({
+                        get: getCloudStorages,
+                        update: updateCloudStorage,
+                        create: createCloudStorage,
+                        delete: deleteCloudStorage,
+                    }),
+                    writable: false,
+                },
             }),
         );
     }