diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 58c3b63113a53b5647d6f9afbb26286f4a553be0..3f72d3de2efe091f180d22fa4d4baef8be7e9f03 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -255,7 +255,8 @@ jobs: specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_objects', 'actions_objects2', 'actions_users', 'actions_projects_models', 'actions_organizations', 'canvas3d_functionality', - 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons'] + 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons', + 'analytics'] steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 023645d523806b6713bdeb38dc42d480814471fa..26b7e961bf809b3022f70b962121e0b4543d0ff6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -270,7 +270,8 @@ jobs: specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_objects', 'actions_objects2', 'actions_users', 'actions_projects_models', 'actions_organizations', 'canvas3d_functionality', - 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons'] + 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons', + 'analytics'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index af521ccae138870333718100e53e5c99a2bea4d5..6d2485be8f454553559e656c91d4892ecee3df1b 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -195,7 +195,8 @@ jobs: specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_objects', 'actions_objects2', 'actions_users', 'actions_projects_models', 'actions_organizations', 'canvas3d_functionality', - 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons'] + 'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons', + 'analytics'] steps: - uses: actions/checkout@v3 diff --git a/cvat-ui/src/components/analytics-page/analytics-page.tsx b/cvat-ui/src/components/analytics-page/analytics-page.tsx index bb0a54d98888d6708aa0e78669731903d6eee615..8da7ff7315748165d2b246f4e2952abe868d73ce 100644 --- a/cvat-ui/src/components/analytics-page/analytics-page.tsx +++ b/cvat-ui/src/components/analytics-page/analytics-page.tsx @@ -233,7 +233,7 @@ function AnalyticsPage(): JSX.Element { ); tabs = ( - + @@ -267,7 +267,7 @@ function AnalyticsPage(): JSX.Element { ); tabs = ( - + diff --git a/cvat-ui/src/components/analytics-page/quality/empty-job.tsx b/cvat-ui/src/components/analytics-page/quality/empty-job.tsx index 02bea25e68773790ef3cddcba20b742cf860d958..96566559542f3ee29a80f3de231bbf72c0eaa759 100644 --- a/cvat-ui/src/components/analytics-page/quality/empty-job.tsx +++ b/cvat-ui/src/components/analytics-page/quality/empty-job.tsx @@ -20,7 +20,7 @@ function EmptyJobComponent(props: Props): JSX.Element { return ( - + No Ground Truth job created yet... diff --git a/cvat-ui/src/components/analytics-page/styles.scss b/cvat-ui/src/components/analytics-page/styles.scss index 0825b0c8ad63468713f293528a8df6ddbd6c02fd..28355d53310ea0a8334875f0ad550b9b20e21fd4 100644 --- a/cvat-ui/src/components/analytics-page/styles.scss +++ b/cvat-ui/src/components/analytics-page/styles.scss @@ -139,7 +139,7 @@ margin-bottom: $grid-unit-size * 3; } -.cvat-job-item-empty-gt { +.cvat-job-empty-ground-truth-item { .ant-card-body { padding: $grid-unit-size * 3; } diff --git a/cvat-ui/src/components/create-job-page/job-form.tsx b/cvat-ui/src/components/create-job-page/job-form.tsx index 0054aba2c58703ea93f69293c6e671d4405ab870..06d37ff791cb897f1ae8bb18bc9b8a1e2246c44c 100644 --- a/cvat-ui/src/components/create-job-page/job-form.tsx +++ b/cvat-ui/src/components/create-job-page/job-form.tsx @@ -161,6 +161,7 @@ function JobForm(props: Props): JSX.Element { rules={[{ required: true, message: 'Please, specify quantity' }]} > - + diff --git a/tests/cypress.config.js b/tests/cypress.config.js index d92d28fb99e624a976429452ee890721c216b002..ffed6824048ec38d577063dc49512e48f56d9196 100644 --- a/tests/cypress.config.js +++ b/tests/cypress.config.js @@ -31,6 +31,7 @@ module.exports = defineConfig({ 'cypress/e2e/issues_prs/**/*.js', 'cypress/e2e/issues_prs2/**/*.js', 'cypress/e2e/actions_users/**/*.js', + 'cypress/e2e/analytics/*.js', 'cypress/e2e/actions_projects_models/**/*.js', 'cypress/e2e/actions_organizations/**/*.js', 'cypress/e2e/remove_users_tasks_projects_organizations.js', diff --git a/tests/cypress/e2e/analytics/ground_truth_jobs.js b/tests/cypress/e2e/analytics/ground_truth_jobs.js new file mode 100644 index 0000000000000000000000000000000000000000..e7b0a8ba3e6e5f839a0b3268af57d0e7555a985d --- /dev/null +++ b/tests/cypress/e2e/analytics/ground_truth_jobs.js @@ -0,0 +1,351 @@ +// Copyright (C) 2023 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/// + +context('Ground truth jobs', () => { + const caseId = 'Ground truth jobs'; + const labelName = 'car'; + const taskName = `Annotation task for Case ${caseId}`; + const attrName = `Attr for Case ${caseId}`; + const textDefaultValue = 'Some default value for type Text'; + const imagesCount = 10; + const imageFileName = 'ground_truth_1'; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const archiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${archiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + + const jobOptions = { + jobType: 'Ground truth', + frameSelectionMethod: 'Random', + fromTaskPage: true, + }; + + const groundTruthRectangles = [ + { + id: 1, + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 250, + firstY: 350, + secondX: 350, + secondY: 450, + }, + { + id: 2, + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 350, + firstY: 450, + secondX: 450, + secondY: 550, + }, + { + id: 3, + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 350, + firstY: 550, + secondX: 450, + secondY: 650, + }, + ]; + + const rectangles = [ + { + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 270, + firstY: 350, + secondX: 370, + secondY: 450, + }, + { + id: 2, + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 350, + firstY: 450, + secondX: 450, + secondY: 550, + }, + { + points: 'By 2 Points', + type: 'Shape', + labelName, + firstX: 130, + firstY: 200, + secondX: 150, + secondY: 250, + }, + ]; + + let groundTruthJobID = null; + let jobID = null; + let taskID = null; + let qualityReportID = null; + + // With seed = 1, frameCount = 3, totalFrames = 10 - predifined ground truth frames are: + const groundTruthFrames = [1, 6, 7]; + + function checkCardValue(className, value) { + cy.get(className) + .should('be.visible') + .within(() => { + cy.get('.cvat-analytics-card-value').should('have.text', value); + }); + } + + function openQualityTab() { + cy.get('.cvat-task-page-actions-button').click(); + cy.get('.cvat-actions-menu') + .should('be.visible') + .find('[role="menuitem"]') + .filter(':contains("View analytics")') + .last() + .click(); + cy.get('.cvat-task-analytics-tabs') + .within(() => { + cy.contains('span', 'Quality').click(); + }); + } + + function checkRectangle(rectangle) { + cy.get(`#cvat_canvas_shape_${rectangle.id}`) + .should('be.visible') + .should('have.class', 'cvat_canvas_ground_truth'); + cy.get(`#cvat-objects-sidebar-state-item-${rectangle.id}`) + .should('be.visible'); + } + + function checkConflicts(type, amount) { + switch (type) { + case 'warning': { + cy.get('.cvat-conflict-warning').should('have.length', amount); + cy.get('.cvat-objects-sidebar-warning-item').should('have.length', amount); + break; + } + case 'error': { + cy.get('.cvat-conflict-error').should('have.length', amount); + cy.get('.cvat-objects-sidebar-conflict-item').should('have.length', amount); + break; + } + default: { + cy.get('.cvat-conflict-warning').should('not.exist'); + cy.get('.cvat-conflict-error').should('not.exist'); + cy.get('.cvat-objects-sidebar-warning-item').should('not.exist'); + cy.get('.cvat-objects-sidebar-conflict-item').should('not.exist'); + } + } + } + + function waitForReport(authKey, rqID) { + cy.request({ + method: 'POST', + url: `/api/quality/reports?rq_id=${rqID}`, + headers: { + Authorization: `Token ${authKey}`, + }, + body: { + task_id: taskID, + }, + }).then((response) => { + if (response.status === 201) { + qualityReportID = response.body.id; + return; + } + waitForReport(authKey, rqID); + }); + } + + before(() => { + cy.visit('auth/login'); + cy.login(); + cy.visit('/tasks'); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName); + cy.openTask(taskName); + cy.url().then((url) => { + taskID = Number(url.split('/').slice(-1)[0].split('?')[0]); + }); + cy.get('.cvat-job-item').first().invoke('attr', 'data-row-id').then((val) => { + jobID = val; + }); + }); + + describe(`Testing case "${caseId}"`, () => { + it('Create ground truth job from task page', () => { + cy.createJob({ + ...jobOptions, + quantity: 15, + }); + cy.url().then((url) => { + groundTruthJobID = Number(url.split('/').slice(-1)[0].split('?')[0]); + + cy.interactMenu('Open the task'); + cy.get('.cvat-job-item').contains('a', `Job #${groundTruthJobID}`) + .parents('.cvat-job-item') + .find('.ant-tag') + .should('have.text', 'Ground truth'); + }); + }); + + it('Delete ground truth job', () => { + cy.deleteJob(groundTruthJobID); + }); + + it('Check quality page, create ground truth job from quality page', () => { + openQualityTab(); + checkCardValue('.cvat-task-mean-annotation-quality', 'N/A'); + checkCardValue('.cvat-task-gt-conflicts', 'N/A'); + checkCardValue('.cvat-task-issues', '0'); + + cy.get('.cvat-job-empty-ground-truth-item') + .should('be.visible') + .within(() => { + cy.contains('button', 'Create new').click(); + }); + cy.createJob({ + ...jobOptions, + frameCount: 3, + seed: 1, + fromTaskPage: false, + }); + + cy.url().then((url) => { + groundTruthJobID = Number(url.split('/').slice(-1)[0].split('?')[0]); + + cy.interactMenu('Open the task'); + openQualityTab(); + cy.get('.cvat-job-item').contains('a', `Job #${groundTruthJobID}`) + .parents('.cvat-job-item') + .find('.ant-tag') + .should('have.text', 'Ground truth'); + }); + }); + + it('Frame navigation in ground truth job', () => { + cy.get('.cvat-job-item').contains('a', `Job #${groundTruthJobID}`).click(); + cy.get('.cvat-spinner').should('not.exist'); + + groundTruthFrames.forEach((frame) => { + cy.checkFrameNum(frame); + cy.get('.cvat-player-next-button').click(); + }); + + cy.checkFrameNum(groundTruthFrames[2]); + }); + + it('Check ground truth annotations in regular job', () => { + cy.interactMenu('Open the task'); + cy.get('.cvat-job-item').contains('a', `Job #${groundTruthJobID}`).click(); + + groundTruthFrames.forEach((frame, index) => { + cy.goCheckFrameNumber(frame); + cy.createRectangle(groundTruthRectangles[index]); + }); + cy.saveJob(); + cy.interactMenu('Finish the job'); + cy.get('.cvat-modal-content-finish-job').within(() => { + cy.contains('button', 'Continue').click(); + }); + + cy.get('.cvat-job-item').contains('a', `Job #${jobID}`).click(); + cy.changeWorkspace('Review'); + + cy.get('.cvat-objects-sidebar-show-ground-truth').click(); + groundTruthFrames.forEach((frame, index) => { + cy.goCheckFrameNumber(frame); + checkRectangle(groundTruthRectangles[index]); + }); + }); + + it('Add annotations to regular job, check quality report', () => { + cy.changeWorkspace('Standard'); + groundTruthFrames.forEach((frame, index) => { + cy.goCheckFrameNumber(frame); + cy.createRectangle(rectangles[index]); + }); + cy.saveJob(); + cy.interactMenu('Open the task'); + + cy.logout(); + cy.getAuthKey().then((res) => { + const authKey = res.body.key; + cy.request({ + method: 'POST', + url: '/api/quality/reports', + headers: { + Authorization: `Token ${authKey}`, + }, + body: { + task_id: taskID, + }, + }).then((response) => { + const rqID = response.body.rq_id; + waitForReport(authKey, rqID); + }); + }); + cy.login(); + cy.visit('/tasks'); + cy.openTask(taskName); + + openQualityTab(); + cy.intercept('GET', '/api/quality/reports**').as('getReport'); + cy.wait('@getReport'); + checkCardValue('.cvat-task-mean-annotation-quality', '50.0%'); + checkCardValue('.cvat-task-gt-conflicts', '3'); + checkCardValue('.cvat-task-issues', '0'); + }); + + it('Check quality report is available for download', () => { + cy.get('.cvat-analytics-download-report-button').click(); + cy.verifyDownload(`quality-report-task_${taskID}-${qualityReportID}.json`); + }); + + it('Conflicts on canvas and sidebar', () => { + cy.get('.cvat-task-job-list').within(() => { + cy.contains('a', `Job #${jobID}`).click(); + }); + cy.get('.cvat-spinner').should('not.exist'); + + cy.changeWorkspace('Review'); + cy.get('.cvat-objects-sidebar-tabs').within(() => { + cy.contains('span', 'Issues').click(); + }); + cy.get('.cvat-objects-sidebar-show-ground-truth').filter(':visible').click(); + + cy.goCheckFrameNumber(groundTruthFrames[0]); + checkConflicts('warning', 1); + + cy.goCheckFrameNumber(groundTruthFrames[1]); + checkConflicts(); + + cy.goCheckFrameNumber(groundTruthFrames[2]); + checkConflicts('error', 2); + }); + + it('Frames with conflicts navigation', () => { + cy.goCheckFrameNumber(0); + + cy.get('.cvat-issues-sidebar-next-frame').click(); + cy.checkFrameNum(groundTruthFrames[0]); + + cy.get('.cvat-issues-sidebar-next-frame').click(); + cy.checkFrameNum(groundTruthFrames[2]); + }); + }); +}); diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 0a95d342af2d4cdbf0b02e0ed178977b190d1523..82fb3c37174c2518b4e98ae81ffd9f7e8a62e252 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -1255,6 +1255,80 @@ Cypress.Commands.add('deleteCloudStorage', (displayName) => { }); }); +Cypress.Commands.add('createJob', (options = { + jobType: 'Ground truth', + frameSelectionMethod: 'Random', + quantity: null, + frameCount: null, + seed: null, + fromTaskPage: true, +}) => { + const { + jobType, + frameSelectionMethod, + quantity, + frameCount, + seed, + fromTaskPage, + } = options; + + if (fromTaskPage) { + cy.get('.cvat-create-job').click({ force: true }); + } + cy.url().should('include', '/jobs/create'); + + cy.get('.cvat-select-job-type').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .first() + .within(() => { + cy.get(`.ant-select-item-option[title="${jobType}"]`).click(); + }); + + cy.get('.cvat-select-frame-selection-method').click(); + cy.get('.ant-select-dropdown') + .not('.ant-select-dropdown-hidden') + .first() + .within(() => { + cy.get(`.ant-select-item-option[title="${frameSelectionMethod}"]`).click(); + }); + + if (quantity) { + cy.get('.cvat-input-frame-quantity').clear().type(quantity); + } else if (frameCount) { + cy.get('.cvat-input-frame-count').clear().type(frameCount); + } + + if (seed) { + cy.get('.cvat-input-seed').clear().type(seed); + } + + cy.contains('button', 'Submit').click(); + + cy.get('.cvat-spinner').should('not.exist'); + cy.url().should('match', /\/tasks\/\d+\/jobs\/\d+/); +}); + +Cypress.Commands.add('deleteJob', (jobID) => { + cy.get('.cvat-job-item').contains('a', `Job #${jobID}`) + .parents('.cvat-job-item') + .find('.cvat-job-item-more-button') + .trigger('mouseover'); + cy.get('.ant-dropdown') + .not('.ant-dropdown-hidden') + .within(() => { + cy.contains('[role="menuitem"]', 'Delete').click(); + }); + cy.get('.cvat-modal-confirm-delete-job') + .should('contain', `The job ${jobID} will be deleted`) + .within(() => { + cy.contains('button', 'Delete').click(); + }); + cy.get('.cvat-job-item').contains('a', `Job #${jobID}`) + .parents('.cvat-job-item') + .should('have.css', 'opacity', '0.5'); +}); + Cypress.Commands.overwrite('visit', (orig, url, options) => { orig(url, options); cy.closeModalUnsupportedPlatform();