From ec3e1f34a4656fad34ee2f1f9c70b6e8aeb0d116 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 28 Dec 2022 12:54:25 +0300 Subject: [PATCH] Better reporting for user limits (#5225) - Added explanatory messages for actions denied for user limits - Fixed few rules and checks - Upgraded OPA version --- .github/workflows/full.yml | 2 +- .github/workflows/helm.yml | 2 + .github/workflows/main.yml | 2 +- .github/workflows/schedule.yml | 2 +- cvat/apps/iam/permissions.py | 932 +++++++++++++----- cvat/apps/iam/rules/cloudstorages.rego | 3 - cvat/apps/iam/rules/limits.rego | 129 +++ cvat/apps/iam/rules/organizations.rego | 5 - cvat/apps/iam/rules/projects.rego | 5 - cvat/apps/iam/rules/tasks.rego | 12 +- cvat/apps/iam/rules/tests/configs/limits.csv | 14 + .../iam/rules/tests/configs/organizations.csv | 3 +- .../apps/iam/rules/tests/configs/projects.csv | 12 +- cvat/apps/iam/rules/tests/configs/tasks.csv | 21 +- .../apps/iam/rules/tests/configs/webhooks.csv | 8 +- .../tests/generators/limits_test.gen.rego.py | 211 ++++ cvat/apps/iam/rules/webhooks.rego | 5 - cvat/apps/iam/tests/test_rest_api.py | 111 ++- cvat/apps/limit_manager/__init__.py | 0 cvat/apps/limit_manager/apps.py | 6 + cvat/apps/limit_manager/core/limits.py | 230 +++++ .../apps/limit_manager/migrations/__init__.py | 0 cvat/apps/limit_manager/models.py | 11 + cvat/settings/base.py | 16 + cvat/settings/testing.py | 18 + docker-compose.yml | 2 +- helm-chart/values.yaml | 4 +- .../contributing/development-environment.md | 2 +- .../en/docs/contributing/running-tests.md | 4 +- tests/python/rest_api/test_limits.py | 560 +++++++++++ tests/python/rest_api/test_projects.py | 86 +- tests/python/shared/fixtures/data.py | 8 + 32 files changed, 2081 insertions(+), 345 deletions(-) create mode 100644 cvat/apps/iam/rules/limits.rego create mode 100644 cvat/apps/iam/rules/tests/configs/limits.csv create mode 100755 cvat/apps/iam/rules/tests/generators/limits_test.gen.rego.py create mode 100644 cvat/apps/limit_manager/__init__.py create mode 100644 cvat/apps/limit_manager/apps.py create mode 100644 cvat/apps/limit_manager/core/limits.py create mode 100644 cvat/apps/limit_manager/migrations/__init__.py create mode 100644 cvat/apps/limit_manager/models.py create mode 100644 tests/python/rest_api/test_limits.py diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml index 7772637f6..a60ccdba7 100644 --- a/.github/workflows/full.yml +++ b/.github/workflows/full.yml @@ -246,7 +246,7 @@ jobs: python cvat/apps/iam/rules/tests/generate_tests.py \ --output-dir cvat/apps/iam/rules/ - curl -L -o opa https://openpolicyagent.org/downloads/v0.34.2/opa_linux_amd64_static + curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static chmod +x ./opa ./opa test cvat/apps/iam/rules diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 6577145ca..28f02c1ec 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -83,6 +83,8 @@ jobs: pip3 install --user -r tests/python/requirements.txt - name: REST API and SDK tests + # We don't have external services in Helm tests, so we ignore corresponding cases + # They are still tested without Helm run: | kubectl cp tests/mounted_file_share/images $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/home/django/share pytest --platform=kube -m "not with_external_services" tests/python diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1e59492d8..c9f6635fc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -213,7 +213,7 @@ jobs: python cvat/apps/iam/rules/tests/generate_tests.py \ --output-dir cvat/apps/iam/rules/ - curl -L -o opa https://openpolicyagent.org/downloads/v0.34.2/opa_linux_amd64_static + curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static chmod +x ./opa ./opa test cvat/apps/iam/rules diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 61bfc638c..05b946cec 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -221,7 +221,7 @@ jobs: - name: OPA tests run: | - curl -L -o opa https://openpolicyagent.org/downloads/v0.34.2/opa_linux_amd64_static + curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static chmod +x ./opa ./opa test cvat/apps/iam/rules diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index bc7e0550b..6836cf0a1 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -3,21 +3,58 @@ # # SPDX-License-Identifier: MIT +from __future__ import annotations from abc import ABCMeta, abstractmethod from collections import namedtuple +from enum import Enum, auto import operator -from rest_framework.exceptions import ValidationError +from typing import Any, List, Optional, Sequence, Tuple, cast + +from attrs import define, field +from rest_framework.exceptions import ValidationError, PermissionDenied import requests from django.conf import settings from django.db.models import Q from rest_framework.permissions import BasePermission -from cvat.apps.webhooks.models import Webhook from cvat.apps.organizations.models import Membership, Organization from cvat.apps.engine.models import Project, Task, Job, Issue +from cvat.apps.limit_manager.core.limits import (CapabilityContext, LimitManager, + Limits, OrgCloudStoragesContext, OrgTasksContext, ProjectWebhooksContext, + OrgCommonWebhooksContext, + TasksInOrgProjectContext, TasksInUserSandboxProjectContext, UserOrgsContext, + UserSandboxCloudStoragesContext, UserSandboxTasksContext) +from cvat.apps.webhooks.models import WebhookTypeChoice + + +class StrEnum(str, Enum): + def __str__(self) -> str: + return self.value + +class RequestNotAllowedError(PermissionDenied): + pass + +@define +class PermissionResult: + allow: bool + reasons: List[str] = field(factory=list) class OpenPolicyAgentPermission(metaclass=ABCMeta): + url: str + user_id: int + group_name: Optional[str] + org_id: Optional[int] + org_owner_id: Optional[int] + org_role: Optional[str] + scope: str + obj: Optional[Any] + + @classmethod + @abstractmethod + def create(cls, request, view, obj) -> Sequence[OpenPolicyAgentPermission]: + ... + @classmethod def create_base_perm(cls, request, view, scope, obj=None, **kwargs): return cls( @@ -76,9 +113,21 @@ class OpenPolicyAgentPermission(metaclass=ABCMeta): def get_resource(self): return None - def __bool__(self): - r = requests.post(self.url, json=self.payload) - return r.json()['result'] + def check_access(self) -> PermissionResult: + response = requests.post(self.url, json=self.payload) + output = response.json()['result'] + + allow = False + reasons = [] + if isinstance(output, dict): + allow = output['allow'] + reasons = output.get('reasons', []) + elif isinstance(output, bool): + allow = output + else: + raise ValueError("Unexpected response format") + + return PermissionResult(allow=allow, reasons=reasons) def filter(self, queryset): url = self.url.replace('/allow', '/filter') @@ -112,6 +161,13 @@ class OpenPolicyAgentPermission(metaclass=ABCMeta): return queryset.filter(q_objects[0]).distinct() class OrganizationPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + DELETE = 'delete' + UPDATE = 'update' + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -128,12 +184,13 @@ class OrganizationPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'create': 'create', - 'destroy': 'delete', - 'partial_update': 'update', - 'retrieve': 'view' + 'list': Scopes.LIST, + 'create': Scopes.CREATE, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, }.get(view.action, None)] def get_resource(self): @@ -149,15 +206,13 @@ class OrganizationPermission(OpenPolicyAgentPermission): 'role': membership.role if membership else None } } - elif self.scope.startswith('create'): + elif self.scope.startswith(__class__.Scopes.CREATE.value): return { 'id': None, 'owner': { 'id': self.user_id }, 'user': { - 'num_resources': Organization.objects.filter( - owner_id=self.user_id).count(), 'role': 'owner' } } @@ -165,6 +220,14 @@ class OrganizationPermission(OpenPolicyAgentPermission): return None class InvitationPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + DELETE = 'delete' + ACCEPT = 'accept' + RESEND = 'resend' + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -183,13 +246,14 @@ class InvitationPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'create': 'create', - 'destroy': 'delete', - 'partial_update': 'accept' if 'accepted' in - request.query_params else 'resend', - 'retrieve': 'view' + 'list': Scopes.LIST, + 'create': Scopes.CREATE, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.ACCEPT if 'accepted' in + request.query_params else Scopes.RESEND, + 'retrieve': Scopes.VIEW, }.get(view.action)] def get_resource(self): @@ -203,7 +267,7 @@ class InvitationPermission(OpenPolicyAgentPermission): 'id': self.obj.membership.organization.id } } - elif self.scope.startswith('create'): + elif self.scope.startswith(__class__.Scopes.CREATE.value): data = { 'owner': { 'id': self.user_id }, 'invitee': { @@ -218,12 +282,23 @@ class InvitationPermission(OpenPolicyAgentPermission): return data class MembershipPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + UPDATE = 'change' + UPDATE_ROLE = 'change:role' + VIEW = 'view' + DELETE = 'delete' + @classmethod def create(cls, request, view, obj): permissions = [] if view.basename == 'membership': for scope in cls.get_scopes(request, view, obj): - self = cls.create_base_perm(request, view, scope, obj) + params = {} + if scope == 'change:role': + params['role'] = request.data.get('role') + + self = cls.create_base_perm(request, view, scope, obj, **params) permissions.append(self) return permissions @@ -234,12 +309,23 @@ class MembershipPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): - return [{ - 'list': 'list', - 'partial_update': 'change:role', - 'retrieve': 'view', - 'destroy': 'delete' - }.get(view.action)] + Scopes = __class__.Scopes + scopes = [] + + scope = { + 'list': Scopes.LIST, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, + 'destroy': Scopes.DELETE, + }.get(view.action) + + if scope == Scopes.UPDATE: + if request.data.get('role') != cast(Membership, obj).role: + scopes.append(Scopes.UPDATE_ROLE) + elif scope: + scopes.append(scope) + + return scopes def get_resource(self): if self.obj: @@ -253,6 +339,12 @@ class MembershipPermission(OpenPolicyAgentPermission): return None class ServerPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + VIEW = 'view' + SEND_EXCEPTION = 'send:exception' + SEND_LOGS = 'send:logs' + LIST_CONTENT = 'list:content' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -269,20 +361,24 @@ class ServerPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'annotation_formats': 'view', - 'about': 'view', - 'plugins': 'view', - 'exception': 'send:exception', - 'logs': 'send:logs', - 'share': 'list:content', - 'advanced_authentication': 'view', + 'annotation_formats': Scopes.VIEW, + 'about': Scopes.VIEW, + 'plugins': Scopes.VIEW, + 'exception': Scopes.SEND_EXCEPTION, + 'logs': Scopes.SEND_LOGS, + 'share': Scopes.LIST_CONTENT, + 'advanced_authentication': Scopes.VIEW, }.get(view.action, None)] def get_resource(self): return None class LogViewerPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -299,8 +395,9 @@ class LogViewerPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'view', + 'list': Scopes.VIEW, }.get(view.action, None)] def get_resource(self): @@ -309,6 +406,12 @@ class LogViewerPermission(OpenPolicyAgentPermission): } class UserPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + VIEW = 'view' + UPDATE = 'update' + DELETE = 'delete' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -325,18 +428,19 @@ class UserPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'self': 'view', - 'retrieve': 'view', - 'partial_update': 'update', - 'destroy': 'delete' + 'list': Scopes.LIST, + 'self': Scopes.VIEW, + 'retrieve': Scopes.VIEW, + 'partial_update': Scopes.UPDATE, + 'destroy': Scopes.DELETE, }.get(view.action)] @classmethod def create_scope_view(cls, request, user_id): obj = namedtuple('User', ['id'])(id=int(user_id)) - return cls(**cls.unpack_context(request), scope='view', obj=obj) + return cls(**cls.unpack_context(request), scope=__class__.Scopes.VIEW, obj=obj) def get_resource(self): data = None @@ -345,7 +449,7 @@ class UserPermission(OpenPolicyAgentPermission): data = { 'id': self.obj.id } - elif self.scope == 'view': # self + elif self.scope == __class__.Scopes.VIEW: # self data = { 'id': self.user_id } @@ -361,6 +465,12 @@ class UserPermission(OpenPolicyAgentPermission): return data class LambdaPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + VIEW = 'view' + CALL_ONLINE = 'call:online' + CALL_OFFLINE = 'call:offline' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -385,20 +495,29 @@ class LambdaPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - ('function', 'list'): 'list', - ('function', 'retrieve'): 'view', - ('function', 'call'): 'call:online', - ('request', 'create'): 'call:offline', - ('request', 'list'): 'call:offline', - ('request', 'retrieve'): 'call:offline', - ('request', 'destroy'): 'call:offline', + ('function', 'list'): Scopes.LIST, + ('function', 'retrieve'): Scopes.VIEW, + ('function', 'call'): Scopes.CALL_ONLINE, + ('request', 'create'): Scopes.CALL_OFFLINE, + ('request', 'list'): Scopes.CALL_OFFLINE, + ('request', 'retrieve'): Scopes.CALL_OFFLINE, + ('request', 'destroy'): Scopes.CALL_OFFLINE, }.get((view.basename, view.action), None)] def get_resource(self): return None class CloudStoragePermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + LIST_CONTENT = 'list:content' + CREATE = 'create' + VIEW = 'view' + UPDATE = 'update' + DELETE = 'delete' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -415,16 +534,17 @@ class CloudStoragePermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'create': 'create', - 'retrieve': 'view', - 'partial_update': 'update', - 'destroy': 'delete', - 'content': 'list:content', - 'preview': 'view', - 'status': 'view', - 'actions': 'view', + 'list': Scopes.LIST, + 'create': Scopes.CREATE, + 'retrieve': Scopes.VIEW, + 'partial_update': Scopes.UPDATE, + 'destroy': Scopes.DELETE, + 'content': Scopes.LIST_CONTENT, + 'preview': Scopes.VIEW, + 'status': Scopes.VIEW, + 'actions': Scopes.VIEW, }.get(view.action)] def get_resource(self): @@ -433,12 +553,8 @@ class CloudStoragePermission(OpenPolicyAgentPermission): data = { 'owner': { 'id': self.user_id }, 'organization': { - 'id': self.org_id + 'id': self.org_id, } if self.org_id is not None else None, - 'user': { - 'num_resources': Organization.objects.filter( - owner=self.user_id).count() - } } elif self.obj: data = { @@ -452,6 +568,22 @@ class CloudStoragePermission(OpenPolicyAgentPermission): return data class ProjectPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + DELETE = 'delete' + UPDATE = 'update' + UPDATE_OWNER = 'update:owner' + UPDATE_ASSIGNEE = 'update:assignee' + UPDATE_DESC = 'update:desc' + UPDATE_ORG = 'update:organization' + VIEW = 'view' + IMPORT_DATASET = 'import:dataset' + EXPORT_ANNOTATIONS = 'export:annotations' + EXPORT_DATASET = 'export:dataset' + EXPORT_BACKUP = 'export:backup' + IMPORT_BACKUP = 'import:backup' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -475,14 +607,6 @@ class ProjectPermission(OpenPolicyAgentPermission): perm = UserPermission.create_scope_view(request, assignee_id) permissions.append(perm) - if 'organization' in request.data: - org_id = request.data.get('organization') - perm = ProjectPermission.create_scope_create(request, org_id) - # We don't create a project, just move it. Thus need to decrease - # the number of resources. - perm.payload['input']['resource']['user']['num_resources'] -= 1 - permissions.append(perm) - return permissions def __init__(self, **kwargs): @@ -491,41 +615,42 @@ class ProjectPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes scope = { - ('list', 'GET'): 'list', - ('create', 'POST'): 'create', - ('destroy', 'DELETE'): 'delete', - ('partial_update', 'PATCH'): 'update', - ('retrieve', 'GET'): 'view', - ('tasks', 'GET'): 'view', - ('dataset', 'POST'): 'import:dataset', - ('append_dataset_chunk', 'HEAD'): 'import:dataset', - ('append_dataset_chunk', 'PATCH'): 'import:dataset', - ('annotations', 'GET'): 'export:annotations', - ('dataset', 'GET'): 'export:dataset', - ('export_backup', 'GET'): 'export:backup', - ('import_backup', 'POST'): 'import:backup', - ('append_backup_chunk', 'PATCH'): 'import:backup', - ('append_backup_chunk', 'HEAD'): 'import:backup', - ('preview', 'GET'): 'view', + ('list', 'GET'): Scopes.LIST, + ('create', 'POST'): Scopes.CREATE, + ('destroy', 'DELETE'): Scopes.DELETE, + ('partial_update', 'PATCH'): Scopes.UPDATE, + ('retrieve', 'GET'): Scopes.VIEW, + ('tasks', 'GET'): Scopes.VIEW, + ('dataset', 'POST'): Scopes.IMPORT_DATASET, + ('append_dataset_chunk', 'HEAD'): Scopes.IMPORT_DATASET, + ('append_dataset_chunk', 'PATCH'): Scopes.IMPORT_DATASET, + ('annotations', 'GET'): Scopes.EXPORT_ANNOTATIONS, + ('dataset', 'GET'): Scopes.EXPORT_DATASET, + ('export_backup', 'GET'): Scopes.EXPORT_BACKUP, + ('import_backup', 'POST'): Scopes.IMPORT_BACKUP, + ('append_backup_chunk', 'PATCH'): Scopes.IMPORT_BACKUP, + ('append_backup_chunk', 'HEAD'): Scopes.IMPORT_BACKUP, + ('preview', 'GET'): Scopes.VIEW, }.get((view.action, request.method)) scopes = [] - if scope == 'update': + if scope == Scopes.UPDATE: if any(k in request.data for k in ('owner_id', 'owner')): owner_id = request.data.get('owner_id') or request.data.get('owner') if owner_id != getattr(obj.owner, 'id', None): - scopes.append(scope + ':owner') + scopes.append(Scopes.UPDATE_OWNER) if any(k in request.data for k in ('assignee_id', 'assignee')): assignee_id = request.data.get('assignee_id') or request.data.get('assignee') if assignee_id != getattr(obj.assignee, 'id', None): - scopes.append(scope + ':assignee') + scopes.append(Scopes.UPDATE_ASSIGNEE) for field in ('name', 'labels', 'bug_tracker'): if field in request.data: - scopes.append(scope + ':desc') + scopes.append(Scopes.UPDATE_DESC) break if 'organization' in request.data: - scopes.append(scope + ':organization') + scopes.append(Scopes.UPDATE_ORG) else: scopes.append(scope) @@ -537,7 +662,7 @@ class ProjectPermission(OpenPolicyAgentPermission): obj = Project.objects.get(id=project_id) except Project.DoesNotExist as ex: raise ValidationError(str(ex)) - return cls(**cls.unpack_context(request), obj=obj, scope='view') + return cls(**cls.unpack_context(request), obj=obj, scope=__class__.Scopes.VIEW) @classmethod def create_scope_create(cls, request, org_id): @@ -563,7 +688,7 @@ class ProjectPermission(OpenPolicyAgentPermission): org_owner_id=getattr(organization.owner, 'id', None) if organization else None, org_role=getattr(membership, 'role', None), - scope='create') + scope=__class__.Scopes.CREATE) def get_resource(self): data = None @@ -576,41 +701,72 @@ class ProjectPermission(OpenPolicyAgentPermission): "id": getattr(self.obj.organization, 'id', None) } } - elif self.scope in ['create', 'import:backup']: + elif self.scope in [__class__.Scopes.CREATE, __class__.Scopes.IMPORT_BACKUP]: data = { "id": None, "owner": { "id": self.user_id }, "assignee": { - "id": self.assignee_id - }, + "id": self.assignee_id, + } if getattr(self, 'assignee_id', None) else None, 'organization': { - "id": self.org_id - }, - "user": { - "num_resources": Project.objects.filter( - owner_id=self.user_id).count() - } + "id": self.org_id, + } if self.org_id else None, } return data class TaskPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + CREATE_IN_PROJECT = 'create@project' + VIEW = 'view' + UPDATE = 'update' + UPDATE_DESC = 'update:desc' + UPDATE_ORGANIZATION = 'update:organization' + UPDATE_ASSIGNEE = 'update:assignee' + UPDATE_PROJECT = 'update:project' + UPDATE_OWNER = 'update:owner' + DELETE = 'delete' + VIEW_ANNOTATIONS = 'view:annotations' + UPDATE_ANNOTATIONS = 'update:annotations' + DELETE_ANNOTATIONS = 'delete:annotations' + IMPORT_ANNOTATIONS = 'import:annotations' + EXPORT_ANNOTATIONS = 'export:annotations' + EXPORT_DATASET = 'export:dataset' + VIEW_METADATA = 'view:metadata' + UPDATE_METADATA = 'update:metadata' + VIEW_DATA = 'view:data' + UPLOAD_DATA = 'upload:data' + IMPORT_BACKUP = 'import:backup' + EXPORT_BACKUP = 'export:backup' + @classmethod def create(cls, request, view, obj): permissions = [] if view.basename == 'task': project_id = request.data.get('project_id') or request.data.get('project') assignee_id = request.data.get('assignee_id') or request.data.get('assignee') + owner = request.data.get('owner_id') or request.data.get('owner') for scope in cls.get_scopes(request, view, obj): - self = cls.create_base_perm(request, view, scope, obj, - project_id=project_id, assignee_id=assignee_id) + params = { 'project_id': project_id, 'assignee_id': assignee_id } + + if scope == __class__.Scopes.UPDATE_ORGANIZATION: + org_id = request.data.get('organization') + if obj is not None and obj.project is not None: + raise ValidationError('Cannot change the organization for ' + 'a task inside a project') + permissions.append(TaskPermission.create_scope_create(request, org_id)) + elif scope == __class__.Scopes.UPDATE_OWNER: + params['owner_id'] = owner + + self = cls.create_base_perm(request, view, scope, obj, **params) permissions.append(self) if view.action == 'jobs': perm = JobPermission.create_scope_list(request) permissions.append(perm) - owner = request.data.get('owner_id') or request.data.get('owner') if owner: perm = UserPermission.create_scope_view(request, owner) permissions.append(perm) @@ -623,18 +779,6 @@ class TaskPermission(OpenPolicyAgentPermission): perm = ProjectPermission.create_scope_view(request, project_id) permissions.append(perm) - if 'organization' in request.data: - org_id = request.data.get('organization') - perm = TaskPermission.create_scope_create(request, org_id) - # We don't create a project, just move it. Thus need to decrease - # the number of resources. - if obj is not None: - perm.payload['input']['resource']['user']['num_resources'] -= 1 - if obj.project is not None: - ValidationError('Cannot change the organization for ' - 'a task inside a project') - permissions.append(perm) - return permissions def __init__(self, **kwargs): @@ -642,75 +786,88 @@ class TaskPermission(OpenPolicyAgentPermission): self.url = settings.IAM_OPA_DATA_URL + '/tasks/allow' @staticmethod - def get_scopes(request, view, obj): + def get_scopes(request, view, obj) -> Scopes: + Scopes = __class__.Scopes scope = { - ('list', 'GET'): 'list', - ('create', 'POST'): 'create', - ('retrieve', 'GET'): 'view', - ('status', 'GET'): 'view', - ('partial_update', 'PATCH'): 'update', - ('update', 'PUT'): 'update', - ('destroy', 'DELETE'): 'delete', - ('annotations', 'GET'): 'view:annotations', - ('annotations', 'PATCH'): 'update:annotations', - ('annotations', 'DELETE'): 'delete:annotations', - ('annotations', 'PUT'): 'update:annotations', - ('annotations', 'POST'): 'import:annotations', - ('append_annotations_chunk', 'PATCH'): 'update:annotations', - ('append_annotations_chunk', 'HEAD'): 'update:annotations', - ('dataset_export', 'GET'): 'export:dataset', - ('metadata', 'GET'): 'view:metadata', - ('metadata', 'PATCH'): 'update:metadata', - ('data', 'GET'): 'view:data', - ('data', 'POST'): 'upload:data', - ('append_data_chunk', 'PATCH'): 'upload:data', - ('append_data_chunk', 'HEAD'): 'upload:data', - ('jobs', 'GET'): 'view', - ('import_backup', 'POST'): 'import:backup', - ('append_backup_chunk', 'PATCH'): 'import:backup', - ('append_backup_chunk', 'HEAD'): 'import:backup', - ('export_backup', 'GET'): 'export:backup', - ('preview', 'GET'): 'view', + ('list', 'GET'): Scopes.LIST, + ('create', 'POST'): Scopes.CREATE, + ('retrieve', 'GET'): Scopes.VIEW, + ('status', 'GET'): Scopes.VIEW, + ('partial_update', 'PATCH'): Scopes.UPDATE, + ('update', 'PUT'): Scopes.UPDATE, + ('destroy', 'DELETE'): Scopes.DELETE, + ('annotations', 'GET'): Scopes.VIEW_ANNOTATIONS, + ('annotations', 'PATCH'): Scopes.UPDATE_ANNOTATIONS, + ('annotations', 'DELETE'): Scopes.DELETE_ANNOTATIONS, + ('annotations', 'PUT'): Scopes.UPDATE_ANNOTATIONS, + ('annotations', 'POST'): Scopes.IMPORT_ANNOTATIONS, + ('append_annotations_chunk', 'PATCH'): Scopes.UPDATE_ANNOTATIONS, + ('append_annotations_chunk', 'HEAD'): Scopes.UPDATE_ANNOTATIONS, + ('dataset_export', 'GET'): Scopes.EXPORT_DATASET, + ('metadata', 'GET'): Scopes.VIEW_METADATA, + ('metadata', 'PATCH'): Scopes.UPDATE_METADATA, + ('data', 'GET'): Scopes.VIEW_DATA, + ('data', 'POST'): Scopes.UPLOAD_DATA, + ('append_data_chunk', 'PATCH'): Scopes.UPLOAD_DATA, + ('append_data_chunk', 'HEAD'): Scopes.UPLOAD_DATA, + ('jobs', 'GET'): Scopes.VIEW, + ('import_backup', 'POST'): Scopes.IMPORT_BACKUP, + ('append_backup_chunk', 'PATCH'): Scopes.IMPORT_BACKUP, + ('append_backup_chunk', 'HEAD'): Scopes.IMPORT_BACKUP, + ('export_backup', 'GET'): Scopes.EXPORT_BACKUP, + ('preview', 'GET'): Scopes.VIEW, }.get((view.action, request.method)) scopes = [] - if scope == 'create': + if scope == Scopes.CREATE: project_id = request.data.get('project_id') or request.data.get('project') if project_id: - scope = scope + '@project' + scope = Scopes.CREATE_IN_PROJECT scopes.append(scope) - elif scope == 'update': + + elif scope == Scopes.UPDATE: if any(k in request.data for k in ('owner_id', 'owner')): owner_id = request.data.get('owner_id') or request.data.get('owner') if owner_id != getattr(obj.owner, 'id', None): - scopes.append(scope + ':owner') + scopes.append(Scopes.UPDATE_OWNER) + if any(k in request.data for k in ('assignee_id', 'assignee')): assignee_id = request.data.get('assignee_id') or request.data.get('assignee') if assignee_id != getattr(obj.assignee, 'id', None): - scopes.append(scope + ':assignee') + scopes.append(Scopes.UPDATE_ASSIGNEE) + if any(k in request.data for k in ('project_id', 'project')): project_id = request.data.get('project_id') or request.data.get('project') if project_id != getattr(obj.project, 'id', None): - scopes.append(scope + ':project') + scopes.append(Scopes.UPDATE_PROJECT) + if any(k in request.data for k in ('name', 'labels', 'bug_tracker', 'subset')): - scopes.append(scope + ':desc') + scopes.append(Scopes.UPDATE_DESC) + if request.data.get('organization'): - scopes.append(scope + ':organization') + scopes.append(Scopes.UPDATE_ORGANIZATION) - elif scope == 'view:annotations': + elif scope == Scopes.VIEW_ANNOTATIONS: if 'format' in request.query_params: - scope = 'export:annotations' + scope = Scopes.EXPORT_ANNOTATIONS scopes.append(scope) - elif scope == 'update:annotations': + + elif scope == Scopes.UPDATE_ANNOTATIONS: if 'format' in request.query_params and request.method == 'PUT': - scope = 'import:annotations' + scope = Scopes.IMPORT_ANNOTATIONS scopes.append(scope) - else: + + elif scope is not None: scopes.append(scope) + else: + # TODO: think if we can protect from missing endpoints + # assert False, "Unknown scope" + pass + return scopes @classmethod @@ -719,7 +876,7 @@ class TaskPermission(OpenPolicyAgentPermission): obj = Task.objects.get(id=task_id) except Task.DoesNotExist as ex: raise ValidationError(str(ex)) - return cls(**cls.unpack_context(request), obj=obj, scope='view:data') + return cls(**cls.unpack_context(request), obj=obj, scope=__class__.Scopes.VIEW_DATA) def get_resource(self): data = None @@ -739,7 +896,11 @@ class TaskPermission(OpenPolicyAgentPermission): }, } if self.obj.project else None } - elif self.scope in ['create', 'create@project', 'import:backup']: + elif self.scope in [ + __class__.Scopes.CREATE, + __class__.Scopes.CREATE_IN_PROJECT, + __class__.Scopes.IMPORT_BACKUP + ]: project = None if self.project_id: try: @@ -760,24 +921,28 @@ class TaskPermission(OpenPolicyAgentPermission): "owner": { "id": getattr(project.owner, 'id', None) }, "assignee": { "id": getattr(project.assignee, 'id', None) }, 'organization': { - "id": getattr(project.organization, 'id', None) - }, - } if project else None, - "user": { - "num_resources": Project.objects.filter( - owner_id=self.user_id).count() - } + "id": getattr(project.organization, 'id', None), + } if project.organization is not None else None, + } if project is not None else None, } return data class WebhookPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + CREATE = 'create' + CREATE_IN_PROJECT = 'create@project' + CREATE_IN_ORG = 'create@organization' + DELETE = 'delete' + UPDATE = 'update' + LIST = 'list' + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] if view.basename == 'webhook': - project_id = request.data.get('project_id') for scope in cls.get_scopes(request, view, obj): self = cls.create_base_perm(request, view, scope, obj, @@ -801,22 +966,23 @@ class WebhookPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes scope = { - ('create', 'POST'): 'create', - ('destroy', 'DELETE'): 'delete', - ('partial_update', 'PATCH'): 'update', - ('update', 'PUT'): 'update', - ('list', 'GET'): 'list', - ('retrieve', 'GET'): 'view', + ('create', 'POST'): Scopes.CREATE, + ('destroy', 'DELETE'): Scopes.DELETE, + ('partial_update', 'PATCH'): Scopes.UPDATE, + ('update', 'PUT'): Scopes.UPDATE, + ('list', 'GET'): Scopes.LIST, + ('retrieve', 'GET'): Scopes.VIEW, }.get((view.action, request.method)) scopes = [] - if scope == 'create': + if scope == Scopes.CREATE: webhook_type = request.data.get('type') - if webhook_type: - scope += f'@{webhook_type}' - scopes.append(scope) - elif scope in ['update', 'delete', 'list', 'view']: + if webhook_type in [m.value for m in WebhookTypeChoice]: + scope = Scopes(str(scope) + f'@{webhook_type}') + scopes.append(scope) + elif scope in [Scopes.UPDATE, Scopes.DELETE, Scopes.LIST, Scopes.VIEW]: scopes.append(scope) return scopes @@ -836,7 +1002,11 @@ class WebhookPermission(OpenPolicyAgentPermission): data['project'] = { 'owner': {'id': getattr(self.obj.project.owner, 'id', None)} } - elif self.scope in ['create@project', 'create@organization']: + elif self.scope in [ + __class__.Scopes.CREATE, + __class__.Scopes.CREATE_IN_PROJECT, + __class__.Scopes.CREATE_IN_ORG + ]: project = None if self.project_id: try: @@ -844,27 +1014,47 @@ class WebhookPermission(OpenPolicyAgentPermission): except Project.DoesNotExist: raise ValidationError(f"Could not find project with provided id: {self.project_id}") - num_resources = Webhook.objects.filter(project=self.project_id).count() if project \ - else Webhook.objects.filter(organization=self.org_id, project=None).count() - data = { 'id': None, 'owner': self.user_id, + 'project': { + 'owner': { + 'id': project.owner.id, + } if project.owner else None, + } if project else None, 'organization': { - 'id': self.org_id - }, - 'num_resources': num_resources - } - - data['project'] = None if project is None else { - 'owner': { - 'id': getattr(project.owner, 'id', None) - }, + 'id': self.org_id, + } if self.org_id is not None else None, + 'user': { + 'id': self.user_id, + } } return data class JobPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + VIEW = 'view' + UPDATE = 'update' + UPDATE_ASSIGNEE = 'update:assignee' + UPDATE_OWNER = 'update:owner' + UPDATE_PROJECT = 'update:project' + UPDATE_STAGE = 'update:stage' + UPDATE_STATE = 'update:state' + UPDATE_DESC = 'update:desc' + DELETE = 'delete' + VIEW_ANNOTATIONS = 'view:annotations' + UPDATE_ANNOTATIONS = 'update:annotations' + DELETE_ANNOTATIONS = 'delete:annotations' + IMPORT_ANNOTATIONS = 'import:annotations' + EXPORT_ANNOTATIONS = 'export:annotations' + EXPORT_DATASET = 'export:dataset' + VIEW_COMMITS = 'view:commits' + VIEW_DATA = 'view:data' + VIEW_METADATA = 'view:metadata' + UPDATE_METADATA = 'update:metadata' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -898,57 +1088,58 @@ class JobPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes scope = { - ('list', 'GET'): 'list', # TODO: need to add the method - ('retrieve', 'GET'): 'view', - ('partial_update', 'PATCH'): 'update', - ('update', 'PUT'): 'update', # TODO: do we need the method? - ('destroy', 'DELETE'): 'delete', - ('annotations', 'GET'): 'view:annotations', - ('dataset_export', 'GET'): 'export:dataset', - ('annotations', 'PATCH'): 'update:annotations', - ('annotations', 'DELETE'): 'delete:annotations', - ('annotations', 'PUT'): 'update:annotations', - ('annotations', 'POST'): 'import:annotations', - ('append_annotations_chunk', 'PATCH'): 'update:annotations', - ('append_annotations_chunk', 'HEAD'): 'update:annotations', - ('data', 'GET'): 'view:data', - ('metadata','GET'): 'view:metadata', - ('metadata','PATCH'): 'update:metadata', - ('issues', 'GET'): 'view', - ('commits', 'GET'): 'view:commits', - ('preview', 'GET'): 'view', + ('list', 'GET'): Scopes.LIST, # TODO: need to add the method + ('retrieve', 'GET'): Scopes.VIEW, + ('partial_update', 'PATCH'): Scopes.UPDATE, + ('update', 'PUT'): Scopes.UPDATE, # TODO: do we need the method? + ('destroy', 'DELETE'): Scopes.DELETE, + ('annotations', 'GET'): Scopes.VIEW_ANNOTATIONS, + ('annotations', 'PATCH'): Scopes.UPDATE_ANNOTATIONS, + ('annotations', 'DELETE'): Scopes.DELETE_ANNOTATIONS, + ('annotations', 'PUT'): Scopes.UPDATE_ANNOTATIONS, + ('annotations', 'POST'): Scopes.IMPORT_ANNOTATIONS, + ('append_annotations_chunk', 'PATCH'): Scopes.UPDATE_ANNOTATIONS, + ('append_annotations_chunk', 'HEAD'): Scopes.UPDATE_ANNOTATIONS, + ('data', 'GET'): Scopes.VIEW_DATA, + ('metadata','GET'): Scopes.VIEW_METADATA, + ('metadata','PATCH'): Scopes.UPDATE_METADATA, + ('issues', 'GET'): Scopes.VIEW, + ('commits', 'GET'): Scopes.VIEW_COMMITS, + ('dataset_export', 'GET'): Scopes.EXPORT_DATASET, + ('preview', 'GET'): Scopes.VIEW, }.get((view.action, request.method)) scopes = [] - if scope == 'update': + if scope == Scopes.UPDATE: if any(k in request.data for k in ('owner_id', 'owner')): owner_id = request.data.get('owner_id') or request.data.get('owner') if owner_id != getattr(obj.owner, 'id', None): - scopes.append(scope + ':owner') + scopes.append(Scopes.UPDATE_OWNER) if any(k in request.data for k in ('assignee_id', 'assignee')): assignee_id = request.data.get('assignee_id') or request.data.get('assignee') if assignee_id != getattr(obj.assignee, 'id', None): - scopes.append(scope + ':assignee') + scopes.append(Scopes.UPDATE_ASSIGNEE) if any(k in request.data for k in ('project_id', 'project')): project_id = request.data.get('project_id') or request.data.get('project') if project_id != getattr(obj.project, 'id', None): - scopes.append(scope + ':project') + scopes.append(Scopes.UPDATE_PROJECT) if 'stage' in request.data: - scopes.append(scope + ':stage') + scopes.append(Scopes.UPDATE_STAGE) if 'state' in request.data: - scopes.append(scope + ':state') + scopes.append(Scopes.UPDATE_STATE) if any(k in request.data for k in ('name', 'labels', 'bug_tracker', 'subset')): - scopes.append(scope + ':desc') - elif scope == 'view:annotations': + scopes.append(Scopes.UPDATE_DESC) + elif scope == Scopes.VIEW_ANNOTATIONS: if 'format' in request.query_params: - scope = 'export:annotations' + scope = Scopes.EXPORT_ANNOTATIONS scopes.append(scope) - elif scope == 'update:annotations': + elif scope == Scopes.UPDATE_ANNOTATIONS: if 'format' in request.query_params and request.method == 'PUT': - scope = 'import:annotations' + scope = Scopes.IMPORT_ANNOTATIONS scopes.append(scope) else: @@ -983,6 +1174,14 @@ class JobPermission(OpenPolicyAgentPermission): return data class CommentPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + CREATE_IN_ISSUE = 'create@issue' + DELETE = 'delete' + UPDATE = 'update' + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -1000,12 +1199,13 @@ class CommentPermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'create': 'create@issue', - 'destroy': 'delete', - 'partial_update': 'update', - 'retrieve': 'view' + 'list': Scopes.LIST, + 'create': Scopes.CREATE_IN_ISSUE, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, }.get(view.action, None)] def get_resource(self): @@ -1046,7 +1246,7 @@ class CommentPermission(OpenPolicyAgentPermission): "id": self.obj.id, "owner": { "id": getattr(self.obj.owner, 'id', None) } }) - elif self.scope.startswith('create'): + elif self.scope.startswith(__class__.Scopes.CREATE): try: db_issue = Issue.objects.get(id=self.issue_id) except Issue.DoesNotExist as ex: @@ -1059,6 +1259,14 @@ class CommentPermission(OpenPolicyAgentPermission): return data class IssuePermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + LIST = 'list' + CREATE = 'create' + CREATE_IN_JOB = 'create@job' + DELETE = 'delete' + UPDATE = 'update' + VIEW = 'view' + @classmethod def create(cls, request, view, obj): permissions = [] @@ -1082,13 +1290,14 @@ class IssuePermission(OpenPolicyAgentPermission): @staticmethod def get_scopes(request, view, obj): + Scopes = __class__.Scopes return [{ - 'list': 'list', - 'create': 'create@job', - 'destroy': 'delete', - 'partial_update': 'update', - 'retrieve': 'view', - 'comments': 'view' + 'list': Scopes.LIST, + 'create': Scopes.CREATE_IN_JOB, + 'destroy': Scopes.DELETE, + 'partial_update': Scopes.UPDATE, + 'retrieve': Scopes.VIEW, + 'comments': Scopes.VIEW, }.get(view.action, None)] def get_resource(self): @@ -1126,7 +1335,7 @@ class IssuePermission(OpenPolicyAgentPermission): "owner": { "id": getattr(self.obj.owner, 'id', None) }, "assignee": { "id": getattr(self.obj.assignee, 'id', None) } }) - elif self.scope.startswith('create'): + elif self.scope.startswith(__class__.Scopes.CREATE): job_id = self.job_id try: db_job = Job.objects.get(id=job_id) @@ -1141,10 +1350,233 @@ class IssuePermission(OpenPolicyAgentPermission): return data +class LimitPermission(OpenPolicyAgentPermission): + @classmethod + def create(cls, request, view, obj): + return [] # There are no basic (unconditional) permissions + + @classmethod + def create_from_scopes(cls, request, view, obj, scopes: List[OpenPolicyAgentPermission]): + scope_to_caps = [ + (scope_handler, cls._prepare_capability_params(scope_handler)) + for scope_handler in scopes + ] + return [ + cls.create_base_perm(request, view, str(scope_handler.scope), obj, + scope_handler=scope_handler, capabilities=capabilities, + ) + for scope_handler, capabilities in scope_to_caps + if capabilities + ] + + def __init__(self, **kwargs): + self.url = settings.IAM_OPA_DATA_URL + '/limits/result' + self.scope_handler: OpenPolicyAgentPermission = kwargs.pop('scope_handler') + self.capabilities: Tuple[Limits, CapabilityContext] = kwargs.pop('capabilities') + super().__init__(**kwargs) + + @classmethod + def get_scopes(cls, request, view, obj): + scopes = [ + (scope_handler, cls._prepare_capability_params(scope_handler)) + for ctx in OpenPolicyAgentPermission.__subclasses__() + if not issubclass(ctx, cls) + for scope_handler in ctx.create(request, view, obj) + ] + return [ + (scope, capabilities) + for scope, capabilities in scopes + if capabilities + ] + + def get_resource(self): + data = {} + limit_manager = LimitManager() + + def _get_capability_status( + capability: Limits, context: Optional[CapabilityContext] + ) -> dict: + status = limit_manager.get_status(limit=capability, context=context) + return { 'used': status.used, 'max': status.max } + + for capability, context in self.capabilities: + data[self._get_capability_name(capability)] = _get_capability_status( + capability=capability, context=context, + ) + + return { 'limits': data } + + @classmethod + def _get_capability_name(cls, capability: Limits) -> str: + return capability.name + + @classmethod + def _prepare_capability_params(cls, scope: OpenPolicyAgentPermission + ) -> List[Tuple[Limits, CapabilityContext]]: + scope_id = (type(scope), scope.scope) + results = [] + + if scope_id in [ + (TaskPermission, TaskPermission.Scopes.CREATE), + (TaskPermission, TaskPermission.Scopes.IMPORT_BACKUP), + ]: + if getattr(scope, 'org_id') is not None: + results.append(( + Limits.ORG_TASKS, + OrgTasksContext(org_id=scope.org_id) + )) + else: + results.append(( + Limits.USER_SANDBOX_TASKS, + UserSandboxTasksContext(user_id=scope.user_id) + )) + + elif scope_id == (TaskPermission, TaskPermission.Scopes.CREATE_IN_PROJECT): + project = Project.objects.get(id=scope.project_id) + + if getattr(project, 'organization') is not None: + results.append(( + Limits.TASKS_IN_ORG_PROJECT, + TasksInOrgProjectContext( + org_id=project.organization.id, + project_id=project.id, + ) + )) + results.append(( + Limits.ORG_TASKS, + OrgTasksContext(org_id=project.organization.id) + )) + else: + results.append(( + Limits.TASKS_IN_USER_SANDBOX_PROJECT, + TasksInUserSandboxProjectContext( + user_id=project.owner.id, + project_id=project.id + ) + )) + results.append(( + Limits.USER_SANDBOX_TASKS, + UserSandboxTasksContext(user_id=project.owner.id) + )) + + elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_PROJECT): + task = cast(Task, scope.obj) + project = Project.objects.get(id=scope.project_id) + + class OwnerType(Enum): + org = auto() + user = auto() + + if getattr(task, 'organization', None): + old_owner = (OwnerType.org, task.organization.id) + else: + old_owner = (OwnerType.user, task.owner.id) + + if getattr(project, 'organization', None) is not None: + results.append(( + Limits.TASKS_IN_ORG_PROJECT, + TasksInOrgProjectContext( + org_id=project.organization.id, + project_id=project.id, + ) + )) + + if old_owner != (OwnerType.org, project.organization.id): + results.append(( + Limits.ORG_TASKS, + OrgTasksContext(org_id=project.organization.id) + )) + else: + results.append(( + Limits.TASKS_IN_USER_SANDBOX_PROJECT, + TasksInUserSandboxProjectContext( + user_id=project.owner.id, + project_id=project.id + ) + )) + + if old_owner != (OwnerType.user, project.owner.id): + results.append(( + Limits.USER_SANDBOX_TASKS, + UserSandboxTasksContext(user_id=project.owner.id) + )) + + elif scope_id == (TaskPermission, TaskPermission.Scopes.UPDATE_OWNER): + task = cast(Task, scope.obj) + + class OwnerType(Enum): + org = auto() + user = auto() + + if getattr(task, 'organization', None) is not None: + old_owner = (OwnerType.org, task.organization.id) + else: + old_owner = (OwnerType.user, task.owner.id) + + new_owner = getattr(scope, 'owner_id', None) + if new_owner is not None and old_owner != (OwnerType.user, new_owner): + results.append(( + Limits.USER_SANDBOX_TASKS, + UserSandboxTasksContext(user_id=new_owner) + )) + + elif scope_id in [ + (ProjectPermission, ProjectPermission.Scopes.CREATE), + (ProjectPermission, ProjectPermission.Scopes.IMPORT_BACKUP), + ]: + if getattr(scope, 'org_id') is not None: + results.append(( + Limits.ORG_PROJECTS, + OrgTasksContext(org_id=scope.org_id) + )) + else: + results.append(( + Limits.USER_SANDBOX_PROJECTS, + UserSandboxTasksContext(user_id=scope.user_id) + )) + + elif scope_id == (CloudStoragePermission, CloudStoragePermission.Scopes.CREATE): + if getattr(scope, 'org_id') is not None: + results.append(( + Limits.ORG_CLOUD_STORAGES, + OrgCloudStoragesContext(org_id=scope.org_id) + )) + else: + results.append(( + Limits.USER_SANDBOX_CLOUD_STORAGES, + UserSandboxCloudStoragesContext(user_id=scope.user_id) + )) + + elif scope_id == (OrganizationPermission, OrganizationPermission.Scopes.CREATE): + results.append(( + Limits.USER_OWNED_ORGS, + UserOrgsContext(user_id=scope.user_id) + )) + + elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_ORG): + results.append(( + Limits.ORG_COMMON_WEBHOOKS, + OrgCommonWebhooksContext(org_id=scope.org_id) + )) + + elif scope_id == (WebhookPermission, WebhookPermission.Scopes.CREATE_IN_PROJECT): + results.append(( + Limits.PROJECT_WEBHOOKS, + ProjectWebhooksContext(project_id=scope.project_id) + )) + + return results + + class PolicyEnforcer(BasePermission): # pylint: disable=no-self-use def check_permission(self, request, view, obj): - permissions = [] + # Some permissions are only needed to be checked if the action + # is permitted in general. To achieve this, we split checks + # into 2 groups, and check one after another. + basic_permissions: List[OpenPolicyAgentPermission] = [] + conditional_permissions: List[OpenPolicyAgentPermission] = [] + # DRF can send OPTIONS request. Internally it will try to get # information about serializers for PUT and POST requests (clone # request and replace the http method). To avoid handling @@ -1152,9 +1584,29 @@ class PolicyEnforcer(BasePermission): # the condition below is enough. if not self.is_metadata_request(request, view): for perm in OpenPolicyAgentPermission.__subclasses__(): - permissions.extend(perm.create(request, view, obj)) - - return all(permissions) + basic_permissions.extend(perm.create(request, view, obj)) + + conditional_permissions.extend(LimitPermission.create_from_scopes( + request, view, obj, basic_permissions + )) + + allow = self._check_permissions(basic_permissions) + if allow and conditional_permissions: + allow = self._check_permissions(conditional_permissions) + return allow + + def _check_permissions(self, permissions: List[OpenPolicyAgentPermission]) -> bool: + allow = True + reasons = [] + for perm in permissions: + result = perm.check_access() + allow &= result.allow + reasons.extend(result.reasons) + + if allow: + return True + else: + raise RequestNotAllowedError(reasons or "not authorized") def has_permission(self, request, view): if not view.detail: diff --git a/cvat/apps/iam/rules/cloudstorages.rego b/cvat/apps/iam/rules/cloudstorages.rego index 7fd1b696f..63c472385 100644 --- a/cvat/apps/iam/rules/cloudstorages.rego +++ b/cvat/apps/iam/rules/cloudstorages.rego @@ -23,9 +23,6 @@ import data.organizations # "id": , # "owner": { "id": }, # "organization": { "id": } or null, -# "user": { -# "num_resources": -# } # } # } diff --git a/cvat/apps/iam/rules/limits.rego b/cvat/apps/iam/rules/limits.rego new file mode 100644 index 000000000..b60403840 --- /dev/null +++ b/cvat/apps/iam/rules/limits.rego @@ -0,0 +1,129 @@ +package limits + +import future.keywords.if +import future.keywords.in +import future.keywords.contains + +import data.utils + + +CAP_USER_SANDBOX_TASKS = "USER_SANDBOX_TASKS" +CAP_USER_SANDBOX_PROJECTS = "USER_SANDBOX_PROJECTS" +CAP_TASKS_IN_USER_SANDBOX_PROJECT = "TASKS_IN_USER_SANDBOX_PROJECT" +CAP_USER_OWNED_ORGS = "USER_OWNED_ORGS" +CAP_USER_SANDBOX_CLOUD_STORAGES = "USER_SANDBOX_CLOUD_STORAGES" +CAP_ORG_TASKS = "ORG_TASKS" +CAP_ORG_PROJECTS = "ORG_PROJECTS" +CAP_TASKS_IN_ORG_PROJECT = "TASKS_IN_ORG_PROJECT" +CAP_ORG_CLOUD_STORAGES = "ORG_CLOUD_STORAGES" +CAP_ORG_COMMON_WEBHOOKS = "ORG_COMMON_WEBHOOKS" +CAP_PROJECT_WEBHOOKS = "PROJECT_WEBHOOKS" + + +check_limit_exceeded(current, max) { + null != max + current >= max +} + + + +problems contains "user tasks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_USER_SANDBOX_TASKS].used, + input.resource.limits[CAP_USER_SANDBOX_TASKS].max + ) +} + +problems contains "user projects limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_USER_SANDBOX_PROJECTS].used, + input.resource.limits[CAP_USER_SANDBOX_PROJECTS].max + ) +} + +problems contains "user project tasks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_TASKS_IN_USER_SANDBOX_PROJECT].used, + input.resource.limits[CAP_TASKS_IN_USER_SANDBOX_PROJECT].max + ) +} + +problems contains "org tasks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_ORG_TASKS].used, + input.resource.limits[CAP_ORG_TASKS].max + ) +} + +problems contains "org projects limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_ORG_PROJECTS].used, + input.resource.limits[CAP_ORG_PROJECTS].max + ) +} + +problems contains "org project tasks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_TASKS_IN_ORG_PROJECT].used, + input.resource.limits[CAP_TASKS_IN_ORG_PROJECT].max + ) +} + +problems contains "project webhooks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_PROJECT_WEBHOOKS].used, + input.resource.limits[CAP_PROJECT_WEBHOOKS].max + ) +} + +problems contains "org webhooks limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_ORG_COMMON_WEBHOOKS].used, + input.resource.limits[CAP_ORG_COMMON_WEBHOOKS].max + ) +} + +problems contains "user orgs limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_USER_OWNED_ORGS].used, + input.resource.limits[CAP_USER_OWNED_ORGS].max + ) +} + +problems contains "user cloud storages limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_USER_SANDBOX_CLOUD_STORAGES].used, + input.resource.limits[CAP_USER_SANDBOX_CLOUD_STORAGES].max + ) +} + +problems contains "org cloud storages limit reached" if { + check_limit_exceeded( + input.resource.limits[CAP_ORG_CLOUD_STORAGES].used, + input.resource.limits[CAP_ORG_CLOUD_STORAGES].max + ) +} + +# In the case of invalid input or no applicable limits, +# we deny the request. We suppose that we always check at least 1 +# limit, and this package is queried by IAM only when there are +# limits to check in the input scope. +default result = { + "allow": false, + "reasons": [] +} + +result := { + "allow": true, + "reasons": [], +} if { + utils.is_admin +} else := { + "allow": count(problems) == 0, + "reasons": problems +} if { + not utils.is_admin + count(input.resource.limits) != 0 +} + +allow := result.allow diff --git a/cvat/apps/iam/rules/organizations.rego b/cvat/apps/iam/rules/organizations.rego index 66b4023d8..10410055e 100644 --- a/cvat/apps/iam/rules/organizations.rego +++ b/cvat/apps/iam/rules/organizations.rego @@ -15,10 +15,6 @@ import data.utils # "owner": { # "id": # }, -# "user": { -# "num_resources": , -# "role": <"owner"|"maintainer"|"supervisor"|"worker"> or null -# } # } # } @@ -68,7 +64,6 @@ allow { allow { input.scope == utils.CREATE - input.resource.user.num_resources == 0 utils.has_perm(utils.USER) } diff --git a/cvat/apps/iam/rules/projects.rego b/cvat/apps/iam/rules/projects.rego index d0af44df3..65677a848 100644 --- a/cvat/apps/iam/rules/projects.rego +++ b/cvat/apps/iam/rules/projects.rego @@ -26,9 +26,6 @@ import data.organizations # "owner": { "id": }, # "assignee": { "id": }, # "organization": { "id": } or null, -# "user": { -# "num_resources": -# } # } # } @@ -49,14 +46,12 @@ allow { allow { { utils.CREATE, utils.IMPORT_BACKUP }[input.scope] utils.is_sandbox - input.resource.user.num_resources < 3 utils.has_perm(utils.USER) } allow { { utils.CREATE, utils.IMPORT_BACKUP }[input.scope] input.auth.organization.id == input.resource.organization.id - input.resource.user.num_resources < 3 utils.has_perm(utils.USER) organizations.has_perm(organizations.SUPERVISOR) } diff --git a/cvat/apps/iam/rules/tasks.rego b/cvat/apps/iam/rules/tasks.rego index 639a8bb2d..0f777d69d 100644 --- a/cvat/apps/iam/rules/tasks.rego +++ b/cvat/apps/iam/rules/tasks.rego @@ -1,4 +1,8 @@ package tasks + +import future.keywords.if +import future.keywords.in + import data.utils import data.organizations @@ -33,9 +37,6 @@ import data.organizations # "assignee": { "id": }, # "organization": { "id": } or null, # } or null, -# "user": { -# "num_resources": -# } # } # } @@ -85,7 +86,6 @@ allow { { utils.CREATE, utils.IMPORT_BACKUP }[input.scope] utils.is_sandbox utils.has_perm(utils.USER) - input.resource.user.num_resources < 10 } allow { @@ -93,7 +93,6 @@ allow { input.auth.organization.id == input.resource.organization.id utils.has_perm(utils.USER) organizations.has_perm(organizations.SUPERVISOR) - input.resource.user.num_resources < 10 } allow { @@ -113,7 +112,6 @@ allow { input.scope == utils.CREATE_IN_PROJECT utils.is_sandbox utils.has_perm(utils.USER) - input.resource.user.num_resources < 10 is_project_staff } @@ -122,7 +120,6 @@ allow { input.auth.organization.id == input.resource.organization.id utils.has_perm(utils.USER) organizations.has_perm(organizations.SUPERVISOR) - input.resource.user.num_resources < 10 } allow { @@ -131,7 +128,6 @@ allow { utils.has_perm(utils.USER) organizations.has_perm(organizations.WORKER) is_project_staff - input.resource.user.num_resources < 10 } allow { diff --git a/cvat/apps/iam/rules/tests/configs/limits.csv b/cvat/apps/iam/rules/tests/configs/limits.csv new file mode 100644 index 000000000..15ce7466f --- /dev/null +++ b/cvat/apps/iam/rules/tests/configs/limits.csv @@ -0,0 +1,14 @@ +TestKind,Capability,CapKind +single,USER_SANDBOX_TASKS,max +single,USER_SANDBOX_PROJECTS,max +single,TASKS_IN_USER_SANDBOX_PROJECT,max +single,USER_OWNED_ORGS,max +single,USER_SANDBOX_CLOUD_STORAGES,max +single,ORG_TASKS,max +single,ORG_PROJECTS,max +single,TASKS_IN_ORG_PROJECT,max +single,ORG_CLOUD_STORAGES,max +single,ORG_COMMON_WEBHOOKS,max +single,PROJECT_WEBHOOKS,max +multi,"USER_SANDBOX_TASKS,USER_SANDBOX_PROJECTS",N/A +multi,,N/A \ No newline at end of file diff --git a/cvat/apps/iam/rules/tests/configs/organizations.csv b/cvat/apps/iam/rules/tests/configs/organizations.csv index e0e389a5f..3e8c985d9 100644 --- a/cvat/apps/iam/rules/tests/configs/organizations.csv +++ b/cvat/apps/iam/rules/tests/configs/organizations.csv @@ -1,6 +1,5 @@ Scope,Resource,Context,Ownership,Limit,Method,URL,Privilege,Membership -create,Organization,N/A,N/A,"resource[""user""][""num_resources""] < 1",POST,/organizations,User,N/A -create,Organization,N/A,N/A,,POST,/organizations,Business,N/A +create,Organization,N/A,N/A,,POST,/organizations,User,N/A list,N/A,N/A,N/A,,GET,/organizations,None,N/A view,Organization,N/A,"Worker, Supervisor, Maintainer, Owner",,GET,/organizations/{id},None,N/A view,Organization,N/A,None,,GET,/organizations/{id},Admin,N/A diff --git a/cvat/apps/iam/rules/tests/configs/projects.csv b/cvat/apps/iam/rules/tests/configs/projects.csv index c68f8439b..bfe3bbff0 100644 --- a/cvat/apps/iam/rules/tests/configs/projects.csv +++ b/cvat/apps/iam/rules/tests/configs/projects.csv @@ -1,12 +1,8 @@ Scope,Resource,Context,Ownership,Limit,Method,URL,Privilege,Membership -create,Project,Sandbox,N/A,resource['user']['num_resources'] < 3,POST,/projects,User,N/A -create,Project,Organization,N/A,resource['user']['num_resources'] < 3,POST,/projects,User,Supervisor -create,Project,Sandbox,N/A,,POST,/projects,Business,N/A -create,Project,Organization,N/A,,POST,/projects,Business,Supervisor -import:backup,Project,Sandbox,N/A,resource['user']['num_resources'] < 3,POST,/projects/backup,User,N/A -import:backup,Project,Organization,N/A,resource['user']['num_resources'] < 3,POST,/projects/backup,User,Supervisor -import:backup,Project,Sandbox,N/A,,POST,/projects/backup,Business,N/A -import:backup,Project,Organization,N/A,,POST,/projects/backup,Business,Supervisor +create,Project,Sandbox,N/A,,POST,/projects,User,N/A +create,Project,Organization,N/A,,POST,/projects,User,Supervisor +import:backup,Project,Sandbox,N/A,,POST,/projects/backup,User,N/A +import:backup,Project,Organization,N/A,,POST,/projects/backup,User,Supervisor list,N/A,Sandbox,N/A,,GET,/projects,None,N/A list,N/A,Organization,N/A,,GET,/projects,None,Worker view,Project,Sandbox,None,,GET,"/projects/{id}, /projects/{id}/tasks",Admin,N/A diff --git a/cvat/apps/iam/rules/tests/configs/tasks.csv b/cvat/apps/iam/rules/tests/configs/tasks.csv index 3e37849d8..a748f5efb 100644 --- a/cvat/apps/iam/rules/tests/configs/tasks.csv +++ b/cvat/apps/iam/rules/tests/configs/tasks.csv @@ -1,19 +1,12 @@ Scope,Resource,Context,Ownership,Limit,Method,URL,Privilege,Membership -create,Task,Sandbox,None,resource['user']['num_resources'] < 10,POST,/tasks,User,N/A -create,Task,Organization,None,resource['user']['num_resources'] < 10,POST,/tasks,User,Supervisor -create,Task,Sandbox,None,,POST,/tasks,Business,N/A -create,Task,Organization,None,,POST,/tasks,Business,Supervisor -import:backup,Task,Sandbox,None,resource['user']['num_resources'] < 10,POST,/tasks/backup,User,N/A -import:backup,Task,Organization,None,resource['user']['num_resources'] < 10,POST,/tasks/backup,User,Supervisor -import:backup,Task,Sandbox,None,,POST,/tasks/backup,Business,N/A -import:backup,Task,Organization,None,,POST,/tasks/backup,Business,Supervisor +create,Task,Sandbox,None,,POST,/tasks,User,N/A +create,Task,Organization,None,,POST,/tasks,User,Supervisor +import:backup,Task,Sandbox,None,,POST,/tasks/backup,User,N/A +import:backup,Task,Organization,None,,POST,/tasks/backup,User,Supervisor create@project,"Task, Project",Sandbox,None,,POST,/tasks,Admin,N/A -create@project,"Task, Project",Sandbox,"Project:owner, Project:assignee",resource['user']['num_resources'] < 10,POST,/tasks,User,N/A -create@project,"Task, Project",Organization,None,resource['user']['num_resources'] < 10,POST,/tasks,User,Supervisor -create@project,"Task, Project",Organization,"Project:owner, Project:assignee",resource['user']['num_resources'] < 10,POST,/tasks,User,Worker -create@project,"Task, Project",Sandbox,"Project:owner, Project:assignee",,POST,/tasks,Business,N/A -create@project,"Task, Project",Organization,None,,POST,/tasks,Business,Supervisor -create@project,"Task, Project",Organization,"Project:owner, Project:assignee",,POST,/tasks,Business,Worker +create@project,"Task, Project",Sandbox,"Project:owner, Project:assignee",,POST,/tasks,User,N/A +create@project,"Task, Project",Organization,None,,POST,/tasks,User,Supervisor +create@project,"Task, Project",Organization,"Project:owner, Project:assignee",,POST,/tasks,User,Worker view,Task,Sandbox,None,,GET,"/tasks/{id}, /tasks/{id}/status",Admin,N/A view,Task,Sandbox,"Owner, Project:owner, Assignee, Project:assignee",,GET,"/tasks/{id}, /tasks/{id}/status",None,N/A view,Task,Organization,None,,GET,"/tasks/{id}, /tasks/{id}/status",User,Maintainer diff --git a/cvat/apps/iam/rules/tests/configs/webhooks.csv b/cvat/apps/iam/rules/tests/configs/webhooks.csv index eb87a418b..5e9198bfe 100644 --- a/cvat/apps/iam/rules/tests/configs/webhooks.csv +++ b/cvat/apps/iam/rules/tests/configs/webhooks.csv @@ -1,10 +1,10 @@ Scope,Resource,Context,Ownership,Limit,Method,URL,Privilege,Membership create@project,Webhook,Sandbox,N/A,,POST,/webhooks,Admin,N/A -create@project,Webhook,Sandbox,Project:owner,resource['num_resources'] < 10,POST,/webhooks,Worker,N/A -create@project,Webhook,Organization,N/A,resource['num_resources'] < 10,POST,/webhooks,Worker,Maintainer -create@project,Webhook,Organization,Project:owner,resource['num_resources'] < 10,POST,/webhooks,Worker,Worker +create@project,Webhook,Sandbox,Project:owner,,POST,/webhooks,Worker,N/A +create@project,Webhook,Organization,N/A,,POST,/webhooks,Worker,Maintainer +create@project,Webhook,Organization,Project:owner,,POST,/webhooks,Worker,Worker create@organization,Webhook,Organization,N/A,,POST,/webhooks,Admin,N/A -create@organization,Webhook,Organization,N/A,resource['num_resources'] < 10,POST,/webhooks,Worker,Maintainer +create@organization,Webhook,Organization,N/A,,POST,/webhooks,Worker,Maintainer update,Webhook,Sandbox,N/A,,PATCH,/webhooks/{id},Admin,N/A update,Webhook,Sandbox,"Project:owner, owner",,PATCH,/webhooks/{id},Worker,N/A update,Webhook,Organization,N/A,,PATCH,/webhooks/{id},Worker,Maintainer diff --git a/cvat/apps/iam/rules/tests/generators/limits_test.gen.rego.py b/cvat/apps/iam/rules/tests/generators/limits_test.gen.rego.py new file mode 100755 index 000000000..d97b6f86a --- /dev/null +++ b/cvat/apps/iam/rules/tests/generators/limits_test.gen.rego.py @@ -0,0 +1,211 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import csv +import json +import os +import sys +import textwrap +from enum import Enum +from itertools import product + +NAME = "limits" + + +class TestKinds(str, Enum): + single = "single" + multi = "multi" + + def __str__(self) -> str: + return self.value.lower() + + +class CapKinds(str, Enum): + max = "max" + + def __str__(self) -> str: + return self.value.lower() + + +def read_test_table(name): + # The table describes positive cases and test configurations + table = [] + with open(os.path.join(sys.argv[1], f"{name}.csv")) as f: + for row in csv.DictReader(f): + table.append(row) + + return table + + +test_table = read_test_table(NAME) + +CAPABILITIES = { + entry["Capability"]: entry["CapKind"] + for entry in test_table + if entry["TestKind"] == TestKinds.single +} +ROLES = ["user", "admin"] + +MAX_CAPABILITY_LIMIT_VALUES = [None, 5] +MAX_CAPABILITY_USED_VALUES = [2, 7] + + +def eval_rule(test_kind, role, capabilities, *, data): + if role == "admin": + return { + "allow": True, + "messages": 0, + } + + allow = True + messages = 0 + for capability in capabilities: + cap_name = capability["name"] + cap_kind = CAPABILITIES[cap_name] + cap_data = data["resource"]["limits"][cap_name] + if cap_kind == CapKinds.max: + cap_allow = (cap_data["max"] is None) or (cap_data["used"] < cap_data["max"]) + messages += not cap_allow + allow &= cap_allow + else: + raise ValueError(f"Unknown capability kind {cap_kind}") + + if not capabilities: + allow = False + messages = 0 + + return { + "allow": allow, + "messages": messages, + } + + +def _get_name(prefix, **kwargs): + name = prefix + for k, v in kwargs.items(): + prefix = "_" + str(k) + if isinstance(v, dict): + if "id" in v: + v = v.copy() + v.pop("id") + if v: + name += _get_name(prefix, **v) + else: + name += "".join( + map( + lambda c: c if c.isalnum() else {"@": "_IN_"}.get(c, "_"), + f"{prefix}_{str(v).upper()}", + ) + ) + + return name + + +def get_name(*args, **kwargs): + return _get_name("test", *args, **kwargs) + + +def generate_capability_cases(capability: str): + capability_kind = CAPABILITIES[capability] + if capability_kind == CapKinds.max: + for used, maximum in product(MAX_CAPABILITY_USED_VALUES, MAX_CAPABILITY_LIMIT_VALUES): + yield {"name": capability, "used": used, "max": maximum} + else: + raise ValueError(f"Unknown capability kind {capability_kind}") + + +def generate_test_data(test_kind, role, capabilities): + data = { + "auth": {"user": {"privilege": role}}, + "resource": { + "limits": {}, + }, + } + + for cap_case in capabilities: + cap_name = cap_case["name"] + cap_kind = CAPABILITIES[cap_case["name"]] + if cap_kind == CapKinds.max: + data["resource"]["limits"][cap_name] = { + "used": cap_case["used"], + "max": cap_case["max"], + } + else: + raise ValueError(f"Unknown capability type {cap_kind}") + + return data + + +def generate_test_cases(): + for config in test_table: + test_kind = config["TestKind"] + if test_kind == TestKinds.single: + capability = config["Capability"] + + for role, cap_case in product(ROLES, generate_capability_cases(capability)): + yield dict(test_kind=test_kind, role=role, capabilities=[cap_case]) + + elif test_kind == TestKinds.multi: + if config["Capability"]: + capabilities = config["Capability"].split(",") + else: + capabilities = [] + + capability_cases = [ + generate_capability_cases(capability) for capability in capabilities + ] + + for params in product(ROLES, *capability_cases): + role = params[0] + cap_case = params[1:] + yield dict(test_kind=test_kind, role=role, capabilities=cap_case) + else: + raise ValueError(f"Unknown test kind {test_kind}") + + +def gen_test_rego(name): + with open(f"{name}_test.gen.rego", "wt") as f: + f.write(f"package {name}\n\n") + + for test_params in generate_test_cases(): + test_data = generate_test_data(**test_params) + test_result = eval_rule(**test_params, data=test_data) + test_name = get_name(**test_params) + f.write( + textwrap.dedent( + """ + {test_name} {{ + r := result with input as {data} + r.allow == {allow} + count(r.reasons) == {messages} + }} + """ + ).format( + test_name=test_name, + allow=str(test_result["allow"]).lower(), + messages=test_result["messages"], + data=json.dumps(test_data), + ) + ) + + # Write the script which is used to generate the file + with open(sys.argv[0]) as this_file: + f.write(f"\n\n# {os.path.split(sys.argv[0])[1]}\n") + for line in this_file: + if line.strip(): + f.write(f"# {line}") + else: + f.write(f"#\n") + + # Write rules which are used to generate the file + with open(os.path.join(sys.argv[1], f"{name}.csv")) as rego_file: + f.write(f"\n\n# {name}.csv\n") + for line in rego_file: + if line.strip(): + f.write(f"# {line}") + else: + f.write(f"#\n") + + +gen_test_rego(NAME) diff --git a/cvat/apps/iam/rules/webhooks.rego b/cvat/apps/iam/rules/webhooks.rego index bf28ecaf9..77cb14f44 100644 --- a/cvat/apps/iam/rules/webhooks.rego +++ b/cvat/apps/iam/rules/webhooks.rego @@ -27,7 +27,6 @@ import data.organizations # "project": { # "owner": { "id": num }, # } or null, -# "num_resources": # } # } # @@ -51,10 +50,8 @@ allow { utils.is_sandbox utils.has_perm(utils.USER) is_project_owner - input.resource.num_resources < 10 } - allow { input.scope == utils.LIST utils.is_sandbox @@ -152,7 +149,6 @@ allow { input.auth.organization.id == input.resource.organization.id utils.has_perm(utils.WORKER) organizations.has_perm(organizations.MAINTAINER) - input.resource.num_resources < 10 } allow { @@ -168,6 +164,5 @@ allow { input.auth.organization.id == input.resource.organization.id utils.has_perm(utils.WORKER) organizations.has_perm(organizations.WORKER) - input.resource.num_resources < 10 is_project_owner } diff --git a/cvat/apps/iam/tests/test_rest_api.py b/cvat/apps/iam/tests/test_rest_api.py index 5d539e8a5..8a309d8af 100644 --- a/cvat/apps/iam/tests/test_rest_api.py +++ b/cvat/apps/iam/tests/test_rest_api.py @@ -1,17 +1,26 @@ # Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2022 CVAT.ai Corporation # # SPDX-License-Identifier: MIT +from contextlib import contextmanager +from itertools import repeat +import itertools +from typing import Sequence, Type +from unittest import mock from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase, APIClient from rest_framework.authtoken.models import Token from django.test import override_settings -from cvat.apps.iam.urls import urlpatterns as iam_url_patterns -from cvat.apps.iam.views import ConfirmEmailViewEx from django.urls import path, re_path from allauth.account.views import EmailVerificationSentView +from cvat.apps.iam.permissions import OpenPolicyAgentPermission, PermissionResult +from cvat.apps.iam.urls import urlpatterns as iam_url_patterns +from cvat.apps.iam.views import ConfirmEmailViewEx +from cvat.apps.engine.models import User + urlpatterns = iam_url_patterns + [ re_path(r'^account-confirm-email/(?P[-:\w]+)/$', ConfirmEmailViewEx.as_view(), @@ -20,6 +29,21 @@ urlpatterns = iam_url_patterns + [ name='account_email_verification_sent'), ] +class ForceLogin: + def __init__(self, user, client): + self.user = user + self.client = client + + def __enter__(self): + if self.user: + self.client.force_login(self.user, backend='django.contrib.auth.backends.ModelBackend') + + return self + + def __exit__(self, exception_type, exception_value, traceback): + if self.user: + self.client.logout() + class UserRegisterAPITestCase(APITestCase): user_data = {'first_name': 'test_first', 'last_name': 'test_last', 'username': 'test_username', @@ -73,3 +97,86 @@ class UserRegisterAPITestCase(APITestCase): self._check_response(response, {'first_name': 'test_first', 'last_name': 'test_last', 'username': 'test_username', 'email': 'test_email@test.com', 'email_verification_required': True, 'key': None}) + +class TestIamApi(APITestCase): + @classmethod + def _make_permission_class(cls, results) -> Type[OpenPolicyAgentPermission]: + + class _TestPerm(OpenPolicyAgentPermission): + def get_resource(self): + return {} + + @classmethod + def create(cls, request, view, obj) -> Sequence[OpenPolicyAgentPermission]: + return [ + cls.create_base_perm(request, view, None, obj, result=result) + for result in results + ] + + def check_access(self) -> PermissionResult: + return PermissionResult(allow=self.result[0], reasons=self.result[1]) + + return _TestPerm + + @classmethod + @contextmanager + def _mock_permissions(cls, *perm_results): + with mock.patch('cvat.apps.iam.permissions.OpenPolicyAgentPermission.__subclasses__', + lambda: [cls._make_permission_class(perm_results)] + ): + yield + + ENDPOINT_WITH_AUTH = '/api/users/self' + + def setUp(self): + self.client = APIClient() + + import sys + sys.modules.pop('cvat.apps.iam.permissions', None) + + @classmethod + def _create_db_users(cls): + cls.user = User.objects.create_user(username="user", password="user") + + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls._create_db_users() + + def test_can_report_denial_reason(self): + expected_reasons = ["hello", "world"] + + with self._mock_permissions((False, expected_reasons)), \ + ForceLogin(user=self.user, client=self.client): + response = self.client.get(self.ENDPOINT_WITH_AUTH) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.json(), expected_reasons) + + def test_can_report_merged_denial_reasons(self): + expected_reasons = [["hello", "world"], ["hi", "there"]] + + with self._mock_permissions(*zip(repeat(False), expected_reasons)), \ + ForceLogin(user=self.user, client=self.client): + response = self.client.get(self.ENDPOINT_WITH_AUTH) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.json(), list(itertools.chain.from_iterable(expected_reasons))) + + def test_can_allow_if_no_permission_matches(self): + with self._mock_permissions(), ForceLogin(user=self.user, client=self.client): + response = self.client.get(self.ENDPOINT_WITH_AUTH) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_can_allow_if_permissions_allow(self): + with self._mock_permissions((True, [])), \ + ForceLogin(user=self.user, client=self.client): + response = self.client.get(self.ENDPOINT_WITH_AUTH) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_can_deny_if_some_permissions_deny(self): + expected_reasons = ["hello"] + + with self._mock_permissions((True, []), (False, expected_reasons)), \ + ForceLogin(user=self.user, client=self.client): + response = self.client.get(self.ENDPOINT_WITH_AUTH) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.json(), expected_reasons) diff --git a/cvat/apps/limit_manager/__init__.py b/cvat/apps/limit_manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvat/apps/limit_manager/apps.py b/cvat/apps/limit_manager/apps.py new file mode 100644 index 000000000..2df4e0f33 --- /dev/null +++ b/cvat/apps/limit_manager/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class LimitManagerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'limit_manager' diff --git a/cvat/apps/limit_manager/core/limits.py b/cvat/apps/limit_manager/core/limits.py new file mode 100644 index 000000000..b7edc44fb --- /dev/null +++ b/cvat/apps/limit_manager/core/limits.py @@ -0,0 +1,230 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from enum import Enum, auto +from typing import Optional, cast + +from attrs import define +from django.conf import settings + +from cvat.apps.engine.models import CloudStorage, Project, Task +from cvat.apps.organizations.models import Organization +from cvat.apps.webhooks.models import Webhook + + +class Limits(Enum): + """ + Represents a capability which has an upper limit, and can be consumed. + + Each capability is also supposed to have a separate CapabilityContext class, + representing its parameters. Different parameter combinations should each have + a different enum member, no member reuse is supposed for different limits. + """ + + # TODO: for a capability with N params, there are O(k^N) + # possible limitation combinations. Not all are meaningful, but even though + # it is quite a large number. Example: + + # A "task create" capability [user_id, org_id, project_id] + # yields the following possible limitations: + # - tasks from the user + # - tasks from the user outside orgs + # - tasks from the user inside orgs + # - tasks from the user in the org + # - tasks from the user in the project + # - tasks in the org + # - tasks in the org projects + # ... + # + # Currently, we will cover all of this with a single request to the limit manager. + # For each meaningful combination class a capability enum entry is supposed. + + USER_SANDBOX_TASKS = auto() + USER_SANDBOX_PROJECTS = auto() + TASKS_IN_USER_SANDBOX_PROJECT = auto() + USER_OWNED_ORGS = auto() + USER_SANDBOX_CLOUD_STORAGES = auto() + + ORG_TASKS = auto() + ORG_PROJECTS = auto() + TASKS_IN_ORG_PROJECT = auto() + ORG_CLOUD_STORAGES = auto() + ORG_COMMON_WEBHOOKS = auto() + + PROJECT_WEBHOOKS = auto() + +class CapabilityContext: + pass + +@define(kw_only=True) +class UserCapabilityContext(CapabilityContext): + user_id: int + +@define(kw_only=True) +class OrgCapabilityContext(CapabilityContext): + org_id: int + +@define(kw_only=True) +class UserSandboxTasksContext(UserCapabilityContext): + pass + +@define(kw_only=True) +class OrgTasksContext(OrgCapabilityContext): + pass + +@define(kw_only=True) +class TasksInUserSandboxProjectContext(UserCapabilityContext): + project_id: int + +@define(kw_only=True) +class TasksInOrgProjectContext(OrgCapabilityContext): + project_id: int + +@define(kw_only=True) +class UserSandboxProjectsContext(UserCapabilityContext): + pass + +@define(kw_only=True) +class OrgProjectsContext(OrgCapabilityContext): + pass + +@define(kw_only=True) +class UserSandboxCloudStoragesContext(UserCapabilityContext): + pass + +@define(kw_only=True) +class OrgCloudStoragesContext(OrgCapabilityContext): + pass + +@define(kw_only=True) +class UserOrgsContext(UserCapabilityContext): + pass + +@define(kw_only=True) +class ProjectWebhooksContext(CapabilityContext): + project_id: int + +@define(kw_only=True) +class OrgCommonWebhooksContext(OrgCapabilityContext): + pass + + +@define(frozen=True) +class LimitStatus: + used: Optional[int] + max: Optional[int] + +class LimitManager: + def get_status(self, + limit: Limits, *, + context: Optional[CapabilityContext] = None, + ) -> LimitStatus: + if limit == Limits.USER_OWNED_ORGS: + assert context is not None + context = cast(UserOrgsContext, context) + + return LimitStatus( + Organization.objects.filter(owner_id=context.user_id).count(), + settings.DEFAULT_LIMITS["USER_OWNED_ORGS"], + ) + + elif limit == Limits.USER_SANDBOX_PROJECTS: + assert context is not None + context = cast(UserSandboxProjectsContext, context) + + return LimitStatus( + # TODO: check about active/removed projects + Project.objects.filter(owner=context.user_id, organization=None).count(), + settings.DEFAULT_LIMITS["USER_SANDBOX_PROJECTS"], + ) + + elif limit == Limits.ORG_PROJECTS: + assert context is not None + context = cast(OrgProjectsContext, context) + + return LimitStatus( + # TODO: check about active/removed projects + Project.objects.filter(organization=context.org_id).count(), + settings.DEFAULT_LIMITS["ORG_PROJECTS"], + ) + + elif limit == Limits.USER_SANDBOX_TASKS: + assert context is not None + context = cast(UserSandboxTasksContext, context) + + return LimitStatus( + # TODO: check about active/removed tasks + Task.objects.filter(owner=context.user_id, organization=None).count(), + settings.DEFAULT_LIMITS["USER_SANDBOX_TASKS"], + ) + + elif limit == Limits.ORG_TASKS: + assert context is not None + context = cast(OrgTasksContext, context) + + return LimitStatus( + # TODO: check about active/removed tasks + Task.objects.filter(organization=context.org_id).count(), + settings.DEFAULT_LIMITS["ORG_TASKS"], + ) + + elif limit == Limits.TASKS_IN_USER_SANDBOX_PROJECT: + assert context is not None + context = cast(TasksInUserSandboxProjectContext, context) + + return LimitStatus( + # TODO: check about active/removed tasks + Task.objects.filter(project=context.project_id).count(), + settings.DEFAULT_LIMITS["TASKS_IN_USER_SANDBOX_PROJECT"] + ) + + elif limit == Limits.TASKS_IN_ORG_PROJECT: + assert context is not None + context = cast(TasksInOrgProjectContext, context) + + return LimitStatus( + # TODO: check about active/removed tasks + Task.objects.filter(project=context.project_id).count(), + settings.DEFAULT_LIMITS["TASKS_IN_ORG_PROJECT"] + ) + + elif limit == Limits.PROJECT_WEBHOOKS: + assert context is not None + context = cast(ProjectWebhooksContext, context) + + return LimitStatus( + # We only limit webhooks per project, not per user + # TODO: think over this limit, maybe we should limit per user + Webhook.objects.filter(project=context.project_id).count(), + settings.DEFAULT_LIMITS["PROJECT_WEBHOOKS"] + ) + + elif limit == Limits.ORG_COMMON_WEBHOOKS: + assert context is not None + context = cast(OrgCommonWebhooksContext, context) + + return LimitStatus( + Webhook.objects.filter(organization=context.org_id, project=None).count(), + settings.DEFAULT_LIMITS["ORG_COMMON_WEBHOOKS"] + ) + + elif limit == Limits.USER_SANDBOX_CLOUD_STORAGES: + assert context is not None + context = cast(UserSandboxCloudStoragesContext, context) + + return LimitStatus( + CloudStorage.objects.filter(owner=context.user_id, organization=None).count(), + settings.DEFAULT_LIMITS["USER_SANDBOX_CLOUD_STORAGES"] + ) + + elif limit == Limits.ORG_CLOUD_STORAGES: + assert context is not None + context = cast(OrgCloudStoragesContext, context) + + return LimitStatus( + CloudStorage.objects.filter(organization=context.org_id).count(), + settings.DEFAULT_LIMITS["ORG_CLOUD_STORAGES"] + ) + + raise NotImplementedError(f"Unknown capability {limit.name}") diff --git a/cvat/apps/limit_manager/migrations/__init__.py b/cvat/apps/limit_manager/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvat/apps/limit_manager/models.py b/cvat/apps/limit_manager/models.py new file mode 100644 index 000000000..cd9a64d4c --- /dev/null +++ b/cvat/apps/limit_manager/models.py @@ -0,0 +1,11 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.db import models + +import cvat.apps.limit_manager.core.limits as core + + +class Limits(core.Limits, models.TextChoices): + pass diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 1f1ecfd70..112881786 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -257,6 +257,22 @@ IAM_OPA_DATA_URL = f'{IAM_OPA_HOST}/v1/data' LOGIN_URL = 'rest_login' LOGIN_REDIRECT_URL = '/' +DEFAULT_LIMITS = { + "USER_SANDBOX_TASKS": 10, + "USER_SANDBOX_PROJECTS": 3, + "TASKS_IN_USER_SANDBOX_PROJECT": 5, + "USER_OWNED_ORGS": 1, + "USER_SANDBOX_CLOUD_STORAGES": 10, + + "ORG_TASKS": 10, + "ORG_PROJECTS": 3, + "TASKS_IN_ORG_PROJECT": 5, + "ORG_CLOUD_STORAGES": 10, + "ORG_COMMON_WEBHOOKS": 20, + + "PROJECT_WEBHOOKS": 10, +} + # ORG settings ORG_INVITATION_CONFIRM = 'No' diff --git a/cvat/settings/testing.py b/cvat/settings/testing.py index 7fc6b8afd..1ebd55e1c 100644 --- a/cvat/settings/testing.py +++ b/cvat/settings/testing.py @@ -64,6 +64,24 @@ PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', ) +# Most of the unit tests don't suppose user limitations +# Can be changed where the limits are supposed. +DEFAULT_LIMITS = { + "USER_SANDBOX_TASKS": None, + "USER_SANDBOX_PROJECTS": None, + "TASKS_IN_USER_SANDBOX_PROJECT": None, + "USER_OWNED_ORGS": None, + "USER_SANDBOX_CLOUD_STORAGES": None, + + "ORG_TASKS": None, + "ORG_PROJECTS": None, + "TASKS_IN_ORG_PROJECT": None, + "ORG_CLOUD_STORAGES": None, + "ORG_COMMON_WEBHOOKS": None, + + "PROJECT_WEBHOOKS": None, +} + # When you run ./manage.py test, Django looks at the TEST_RUNNER setting to # determine what to do. By default, TEST_RUNNER points to # 'django.test.runner.DiscoverRunner'. This class defines the default Django diff --git a/docker-compose.yml b/docker-compose.yml index ed81db582..6338fdee7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -187,7 +187,7 @@ services: cvat_opa: container_name: cvat_opa - image: openpolicyagent/opa:0.34.2-rootless + image: openpolicyagent/opa:0.45.0-rootless restart: always networks: cvat: diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index b6d083e4b..a37d6fd53 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -137,8 +137,8 @@ cvat: opa: replicas: 1 image: openpolicyagent/opa - tag: 0.34.2-rootless - imagePullPolicy: IfNotPresent + tag: 0.45.0-rootless + imagepullploicy: IfNotPresent labels: {} # test: test annotations: {} diff --git a/site/content/en/docs/contributing/development-environment.md b/site/content/en/docs/contributing/development-environment.md index c20c5dda6..5220d1318 100644 --- a/site/content/en/docs/contributing/development-environment.md +++ b/site/content/en/docs/contributing/development-environment.md @@ -170,7 +170,7 @@ description: 'Installing a development environment for different operating syste - Pull and run OpenPolicyAgent Docker image: ```bash - docker run -d --rm --name cvat_opa_debug -p 8181:8181 openpolicyagent/opa:0.34.2-rootless \ + docker run -d --rm --name cvat_opa_debug -p 8181:8181 openpolicyagent/opa:0.45.0-rootless \ run --server --set=decision_logs.console=true --set=services.cvat.url=http://host.docker.internal:7000 \ --set=bundles.cvat.service=cvat --set=bundles.cvat.resource=/api/auth/rules ``` diff --git a/site/content/en/docs/contributing/running-tests.md b/site/content/en/docs/contributing/running-tests.md index 7c870fbb6..5f72710c5 100644 --- a/site/content/en/docs/contributing/running-tests.md +++ b/site/content/en/docs/contributing/running-tests.md @@ -162,13 +162,13 @@ python cvat/apps/iam/rules/tests/generate_tests.py \ - In a Docker container ```bash docker run --rm -v ${PWD}/cvat/apps/iam/rules:/rules \ - openpolicyagent/opa:0.34.2-rootless \ + openpolicyagent/opa:0.45.0-rootless \ test /rules -v ``` - or execute OPA directly ```bash -curl -L -o opa https://openpolicyagent.org/downloads/v0.34.2/opa_linux_amd64_static +curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static chmod +x ./opa ./opa test cvat/apps/iam/rules ``` diff --git a/tests/python/rest_api/test_limits.py b/tests/python/rest_api/test_limits.py new file mode 100644 index 000000000..61fed5ddb --- /dev/null +++ b/tests/python/rest_api/test_limits.py @@ -0,0 +1,560 @@ +# Copyright (C) 2022 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import json +from contextlib import contextmanager +from functools import partial +from http import HTTPStatus +from pathlib import Path +from typing import Optional +from uuid import uuid4 + +import boto3 +import pytest +from cvat_sdk import Client, exceptions +from cvat_sdk.api_client import ApiClient, models +from cvat_sdk.core.client import Config +from cvat_sdk.core.proxies.projects import Project +from cvat_sdk.core.proxies.tasks import ResourceType, Task + +from shared.utils.config import ( + BASE_URL, + MINIO_ENDPOINT_URL, + MINIO_KEY, + MINIO_SECRET_KEY, + USER_PASS, + post_method, +) +from shared.utils.helpers import generate_image_file + + +@pytest.fixture +def fxt_image_file(tmp_path: Path): + img_path = tmp_path / "img.png" + with img_path.open("wb") as f: + f.write(generate_image_file(filename=str(img_path), size=(5, 10)).getvalue()) + + return img_path + + +def get_common_storage_params(): + return { + "provider_type": "AWS_S3_BUCKET", + "credentials_type": "KEY_SECRET_KEY_PAIR", + "key": "minio_access_key", + "secret_key": "minio_secret_key", + "specific_attributes": "endpoint_url=http://minio:9000", + } + + +def define_s3_client(): + s3 = boto3.resource( + "s3", + aws_access_key_id=MINIO_KEY, + aws_secret_access_key=MINIO_SECRET_KEY, + endpoint_url=MINIO_ENDPOINT_URL, + ) + return s3.meta.client + + +class TestUserLimits: + @classmethod + def _create_user(cls, api_client: ApiClient, email: str) -> str: + username = email.split("@", maxsplit=1)[0] + with api_client: + (user, _) = api_client.auth_api.create_register( + models.RegisterSerializerExRequest( + username=username, password1=USER_PASS, password2=USER_PASS, email=email + ) + ) + + api_client.cookies.clear() + + return user.username + + def _make_client(self) -> Client: + return Client(BASE_URL, config=Config(status_check_period=0.01)) + + @pytest.fixture(autouse=True) + def setup(self, restore_db_per_function, tmp_path: Path, fxt_image_file: Path): + self.tmp_dir = tmp_path + self.image_file = fxt_image_file + + self.client = self._make_client() + self.user = self._create_user(self.client.api_client, email="test_user_limits@localhost") + + with self.client: + self.client.login((self.user, USER_PASS)) + + @pytest.fixture + def fxt_another_client(self) -> Client: + client = self._make_client() + user = self._create_user(self.client.api_client, email="test_user_limits2@localhost") + + with client: + client.login((user, USER_PASS)) + yield client + + _DEFAULT_TASKS_LIMIT = 10 + _DEFAULT_PROJECT_TASKS_LIMIT = 5 + _DEFAULT_PROJECTS_LIMIT = 3 + _DEFAULT_ORGS_LIMIT = 1 + _DEFAULT_CLOUD_STORAGES_LIMIT = 10 + + _TASK_LIMIT_MESSAGE = "user tasks limit reached" + _PROJECT_TASK_LIMIT_MESSAGE = "user project tasks limit reached" + _PROJECTS_LIMIT_MESSAGE = "user projects limit reached" + _ORGS_LIMIT_MESSAGE = "user orgs limit reached" + _CLOUD_STORAGES_LIMIT_MESSAGE = "user cloud storages limit reached" + + def _create_task( + self, *, project: Optional[int] = None, client: Optional[Client] = None + ) -> Task: + if client is None: + client = self.client + + return client.tasks.create_from_data( + spec=models.TaskWriteRequest( + name="test_task", + labels=[models.PatchedLabelRequest(name="cat")] if not project else [], + project_id=project, + ), + resource_type=ResourceType.LOCAL, + resources=[str(self.image_file)], + ) + + def _create_project(self, *, client: Optional[Client] = None) -> Project: + if client is None: + client = self.client + + return client.projects.create(models.ProjectWriteRequest(name="test_project")) + + def test_can_reach_tasks_limit(self): + for _ in range(self._DEFAULT_TASKS_LIMIT): + self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_importing_backup(self): + for _ in range(self._DEFAULT_TASKS_LIMIT): + task = self._create_task() + + backup_filename = self.tmp_dir / "task_backup.zip" + task.download_backup(backup_filename) + + with pytest.raises(exceptions.ApiException) as capture: + self.client.tasks.create_from_backup(backup_filename) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_project(self): + project = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project) + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task(project=project) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECT_TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_different_projects(self): + project1 = self._create_project().id + project2 = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project1) + for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project2) + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_filled_project(self): + project = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project) + for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task(project=project) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == { + self._TASK_LIMIT_MESSAGE, + self._PROJECT_TASK_LIMIT_MESSAGE, + } + + def test_can_reach_project_tasks_limit_when_moving_into_filled_project(self): + project = self._create_project().id + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project) + + task = self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + task.update(models.PatchedTaskWriteRequest(project_id=project)) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECT_TASK_LIMIT_MESSAGE} + + @pytest.mark.xfail( + raises=AssertionError, reason="only admins can change ownership, but they ignore limits" + ) + def test_can_reach_tasks_limit_when_giving_away_to_another_user( + self, fxt_another_client: Client + ): + for _ in range(self._DEFAULT_TASKS_LIMIT): + self._create_task(client=fxt_another_client) + + task = self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + task.update( + models.PatchedTaskWriteRequest( + owner_id=fxt_another_client.users.retrieve_current_user().id + ) + ) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECT_TASK_LIMIT_MESSAGE} + + @pytest.mark.xfail( + raises=AssertionError, reason="only admins can change ownership, but they ignore limits" + ) + def test_can_reach_project_tasks_limit_when_giving_away_to_another_users_filled_project( + self, fxt_another_client: Client + ): + project = self._create_project(client=fxt_another_client).id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(client=fxt_another_client, project=project) + for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(client=fxt_another_client) + + task = self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + task.update( + models.PatchedTaskWriteRequest( + owner_id=fxt_another_client.users.retrieve_current_user().id + ) + ) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == { + self._DEFAULT_TASKS_LIMIT, + self._PROJECT_TASK_LIMIT_MESSAGE, + } + + @pytest.mark.xfail( + raises=AssertionError, reason="only admins can change ownership, but they ignore limits" + ) + def test_can_reach_projects_limit_when_giving_away_to_another_user( + self, fxt_another_client: Client + ): + for _ in range(self._DEFAULT_PROJECTS_LIMIT): + self._create_project(client=fxt_another_client) + + project = self._create_project() + + with pytest.raises(exceptions.ApiException) as capture: + project.update( + models.PatchedProjectWriteRequest( + owner_id=fxt_another_client.users.retrieve_current_user().id + ) + ) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECT_TASK_LIMIT_MESSAGE} + + def test_can_reach_projects_limit(self): + for _ in range(self._DEFAULT_PROJECTS_LIMIT): + self._create_project() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_project() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECTS_LIMIT_MESSAGE} + + def test_can_reach_projects_limit_when_importing_backup(self): + for _ in range(self._DEFAULT_PROJECTS_LIMIT): + project = self._create_project() + + backup_filename = self.tmp_dir / (project.name + "_backup.zip") + project.download_backup(backup_filename) + + with pytest.raises(exceptions.ApiException) as capture: + self.client.projects.create_from_backup(backup_filename) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECTS_LIMIT_MESSAGE} + + def test_can_reach_orgs_limit(self): + for i in range(self._DEFAULT_ORGS_LIMIT): + (_, response) = self.client.api_client.organizations_api.create( + models.OrganizationWriteRequest(slug=f"test_user_orgs_{i}"), _parse_response=False + ) + assert response.status == HTTPStatus.CREATED + + with pytest.raises(exceptions.ApiException) as capture: + self.client.api_client.organizations_api.create( + models.OrganizationWriteRequest(slug=f"test_user_orgs_{i}"), _parse_response=False + ) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._ORGS_LIMIT_MESSAGE} + + @pytest.mark.with_external_services + def test_can_reach_cloud_storages_limit(self, request: pytest.FixtureRequest): + storage_params = get_common_storage_params() + + # TODO: refactor after https://github.com/opencv/cvat/pull/4819 + s3_client = define_s3_client() + + def _create_bucket(name: str) -> str: + name = name + str(uuid4()) + s3_client.create_bucket(Bucket=name) + request.addfinalizer(partial(s3_client.delete_bucket, Bucket=name)) + return name + + def _add_storage(idx: int): + response = post_method( + self.user, + "cloudstorages", + { + "display_name": f"test_storage{idx}", + "resource": _create_bucket(f"testbucket{idx}"), + **storage_params, + }, + ) + return response + + for i in range(self._DEFAULT_CLOUD_STORAGES_LIMIT): + response = _add_storage(i) + assert response.status_code == HTTPStatus.CREATED + + response = _add_storage(i) + + assert response.status_code == HTTPStatus.FORBIDDEN + assert set(response.json()) == {self._CLOUD_STORAGES_LIMIT_MESSAGE} + + +class TestOrgLimits: + @classmethod + def _create_org(cls, api_client: ApiClient) -> str: + with api_client: + (_, response) = api_client.organizations_api.create( + models.OrganizationWriteRequest(slug="test_org_limits"), _parse_response=False + ) + + return json.loads(response.data) + + def _make_client(self) -> Client: + return Client(BASE_URL, config=Config(status_check_period=0.01)) + + @pytest.fixture(autouse=True) + def setup( + self, restore_db_per_function, tmp_path: Path, regular_user: str, fxt_image_file: Path + ): + self.tmp_dir = tmp_path + self.image_file = fxt_image_file + + self.client = self._make_client() + self.user = regular_user + + with self.client: + self.client.login((self.user, USER_PASS)) + + org = self._create_org(self.client.api_client) + self.org = org["id"] + self.org_slug = org["slug"] + + with self._patch_client_with_org(self.client): + yield + + _DEFAULT_TASKS_LIMIT = 10 + _DEFAULT_PROJECT_TASKS_LIMIT = 5 + _DEFAULT_PROJECTS_LIMIT = 3 + _DEFAULT_CLOUD_STORAGES_LIMIT = 10 + + _TASK_LIMIT_MESSAGE = "org tasks limit reached" + _PROJECT_TASK_LIMIT_MESSAGE = "org project tasks limit reached" + _PROJECTS_LIMIT_MESSAGE = "org projects limit reached" + _CLOUD_STORAGES_LIMIT_MESSAGE = "org cloud storages limit reached" + + @contextmanager + def _patch_client_with_org(self, client: Optional[Client] = None): + if client is None: + client = self.client + + new_headers = self.client.api_client.default_headers.copy() + new_headers["X-Organization"] = self.org_slug + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(client.api_client, "default_headers", new_headers) + yield client + + @pytest.fixture + def fxt_patch_client_with_org(self): + with self._patch_client_with_org(self.client): + yield + + def _create_task( + self, *, project: Optional[int] = None, client: Optional[Client] = None + ) -> Task: + if client is None: + client = self.client + + return client.tasks.create_from_data( + spec=models.TaskWriteRequest( + name="test_task", + labels=[models.PatchedLabelRequest(name="cat")] if not project else [], + project_id=project, + ), + resource_type=ResourceType.LOCAL, + resources=[str(self.image_file)], + ) + + def _create_project(self, *, client: Optional[Client] = None) -> Project: + if client is None: + client = self.client + + return client.projects.create(models.ProjectWriteRequest(name="test_project")) + + def test_can_reach_tasks_limit(self): + for _ in range(self._DEFAULT_TASKS_LIMIT): + self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_importing_backup(self): + for _ in range(self._DEFAULT_TASKS_LIMIT): + task = self._create_task() + + backup_filename = self.tmp_dir / "task_backup.zip" + task.download_backup(backup_filename) + + with pytest.raises(exceptions.ApiException) as capture: + self.client.tasks.create_from_backup(backup_filename) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_project(self): + project = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project) + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task(project=project) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECT_TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_different_projects(self): + project1 = self._create_project().id + project2 = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project1) + for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project2) + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._TASK_LIMIT_MESSAGE} + + def test_can_reach_tasks_limit_when_creating_in_filled_project(self): + project = self._create_project().id + + for _ in range(self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task(project=project) + for _ in range(self._DEFAULT_TASKS_LIMIT - self._DEFAULT_PROJECT_TASKS_LIMIT): + self._create_task() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_task(project=project) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == { + self._TASK_LIMIT_MESSAGE, + self._PROJECT_TASK_LIMIT_MESSAGE, + } + + def test_can_reach_projects_limit(self): + for _ in range(self._DEFAULT_PROJECTS_LIMIT): + self._create_project() + + with pytest.raises(exceptions.ApiException) as capture: + self._create_project() + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECTS_LIMIT_MESSAGE} + + def test_can_reach_projects_limit_when_importing_backup(self): + for _ in range(self._DEFAULT_PROJECTS_LIMIT): + project = self._create_project() + + backup_filename = self.tmp_dir / "test_project_backup.zip" + project.download_backup(str(backup_filename)) + + with pytest.raises(exceptions.ApiException) as capture: + self.client.projects.create_from_backup(str(backup_filename)) + + assert capture.value.status == HTTPStatus.FORBIDDEN + assert set(json.loads(capture.value.body)) == {self._PROJECTS_LIMIT_MESSAGE} + + @pytest.mark.with_external_services + def test_can_reach_cloud_storages_limit(self, request: pytest.FixtureRequest): + storage_params = get_common_storage_params() + + # TODO: refactor after https://github.com/opencv/cvat/pull/4819 + s3_client = define_s3_client() + + def _create_bucket(name: str) -> str: + name = name + str(uuid4()) + s3_client.create_bucket(Bucket=name) + request.addfinalizer(partial(s3_client.delete_bucket, Bucket=name)) + return name + + def _add_storage(idx: int): + response = post_method( + self.user, + "cloudstorages", + { + "display_name": f"test_storage{idx}", + "resource": _create_bucket(f"testbucket{idx}"), + **storage_params, + }, + org_id=self.org, + ) + return response + + for i in range(self._DEFAULT_CLOUD_STORAGES_LIMIT): + response = _add_storage(i) + assert response.status_code == HTTPStatus.CREATED + + response = _add_storage(i) + + assert response.status_code == HTTPStatus.FORBIDDEN + assert set(response.json()) == {self._CLOUD_STORAGES_LIMIT_MESSAGE} diff --git a/tests/python/rest_api/test_projects.py b/tests/python/rest_api/test_projects.py index dd9462df6..fdfbf1ac5 100644 --- a/tests/python/rest_api/test_projects.py +++ b/tests/python/rest_api/test_projects.py @@ -10,13 +10,15 @@ import zipfile from copy import deepcopy from http import HTTPStatus from io import BytesIO -from itertools import groupby, product +from itertools import product from time import sleep +from typing import Dict, Optional import pytest +from cvat_sdk.api_client import ApiClient, Configuration, models from deepdiff import DeepDiff -from shared.utils.config import get_method, make_api_client, patch_method +from shared.utils.config import BASE_URL, USER_PASS, get_method, make_api_client, patch_method from .utils import export_dataset @@ -300,39 +302,7 @@ class TestPostProjects: spec = {"name": f"test {username} tries to create a project"} self._test_create_project_201(username, spec) - def test_if_user_cannot_have_more_than_3_projects(self, projects, find_users): - users = find_users(privilege="user") - - user_id, user_projects = next( - (user_id, len(list(projects))) - for user_id, projects in groupby(projects, lambda a: a["owner"]["id"]) - if len(list(projects)) < 3 - ) - user = users[user_id] - - for i in range(1, 4 - user_projects): - spec = { - "name": f'test: {user["username"]} tries to create a project number {user_projects + i}' - } - self._test_create_project_201(user["username"], spec) - - spec = {"name": f'test {user["username"]} tries to create more than 3 projects'} - self._test_create_project_403(user["username"], spec) - - @pytest.mark.parametrize("privilege", ("admin", "business")) - def test_if_user_can_have_more_than_3_projects(self, find_users, privilege): - privileged_users = find_users(privilege=privilege) - assert len(privileged_users) - - user = privileged_users[0] - - for i in range(1, 5): - spec = { - "name": f'test: {user["username"]} with privilege {privilege} tries to create a project number {i}' - } - self._test_create_project_201(user["username"], spec) - - def test_if_org_worker_cannot_crate_project(self, find_users): + def test_if_org_worker_cannot_create_project(self, find_users): workers = find_users(role="worker") worker = next(u for u in workers if u["org"]) @@ -343,16 +313,52 @@ class TestPostProjects: self._test_create_project_403(worker["username"], spec, org_id=worker["org"]) @pytest.mark.parametrize("role", ("supervisor", "maintainer", "owner")) - def test_if_org_role_can_create_project(self, find_users, role): - privileged_users = find_users(role=role) - assert len(privileged_users) + def test_if_org_role_can_create_project(self, role, admin_user): + # We can hit org or user limits here, so we create a new org and users + user = self._create_user( + ApiClient(configuration=Configuration(BASE_URL)), email="test_org_roles@localhost" + ) - user = next(u for u in privileged_users if u["org"]) + if role != "owner": + org = self._create_org(make_api_client(admin_user), members={user["email"]: role}) + else: + org = self._create_org(make_api_client(user["username"])) spec = { "name": f'test: worker {user["username"]} creating a project for his organization', } - self._test_create_project_201(user["username"], spec, org_id=user["org"]) + self._test_create_project_201(user["username"], spec, org_id=org) + + @classmethod + def _create_user(cls, api_client: ApiClient, email: str) -> str: + username = email.split("@", maxsplit=1)[0] + with api_client: + (_, response) = api_client.auth_api.create_register( + models.RegisterSerializerExRequest( + username=username, password1=USER_PASS, password2=USER_PASS, email=email + ) + ) + + api_client.cookies.clear() + + return json.loads(response.data) + + @classmethod + def _create_org(cls, api_client: ApiClient, members: Optional[Dict[str, str]] = None) -> str: + with api_client: + (_, response) = api_client.organizations_api.create( + models.OrganizationWriteRequest(slug="test_org_roles"), _parse_response=False + ) + org = json.loads(response.data)["id"] + + for email, role in (members or {}).items(): + api_client.invitations_api.create( + models.InvitationWriteRequest(role=role, email=email), + org_id=org, + _parse_response=False, + ) + + return org def _check_cvat_for_video_project_annotations_meta(content, values_to_be_checked): diff --git a/tests/python/shared/fixtures/data.py b/tests/python/shared/fixtures/data.py index 32e1aa2ec..47c603827 100644 --- a/tests/python/shared/fixtures/data.py +++ b/tests/python/shared/fixtures/data.py @@ -378,3 +378,11 @@ def admin_user(users): if user["is_superuser"] and user["is_active"]: return user["username"] raise Exception("Can't find any admin user in the test DB") + + +@pytest.fixture(scope="session") +def regular_user(users): + for user in users: + if not user["is_superuser"] and user["is_active"]: + return user["username"] + raise Exception("Can't find any regular user in the test DB") -- GitLab