diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c17cfc82bba52cd54d98cb2a3a88216a1824981..041f86f5b464b00da52742c04befa90e3816b99b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () - Hide analytics link from non-admin users () - Hide notifications on login/logout/register () +- CVAT and CVAT SDK now use a custom `User-Agent` header in HTTP requests + () ### Deprecated - TBD diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index e64734ca20fa0d5c1ca5a4346db2f42eb2731149..27956094217b03749bf5af0739f99432e213d6a1 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -28,6 +28,7 @@ docker run --rm -v "$DST_DIR:/local" -u "$(id -u)":"$(id -g)" \ -i "/local/schema/schema.yml" \ --config "/local/${TEMPLATE_DIR_NAME}/generator-config.yml" \ -p "packageVersion=$VERSION" \ + -p "httpUserAgent=cvat_sdk/$VERSION" \ -g python \ -o "/local/" diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index a92de62e4dadf609fbe8c9a916b21a7b28eae4b0..64c3d7050a733a69d5adb4224f02d04ffa79a3ed 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -16,7 +16,6 @@ import shutil from distutils.dir_util import copy_tree from urllib import parse as urlparse from urllib import request as urlrequest -import requests import ipaddress import dns.resolver import django_rq @@ -31,6 +30,7 @@ from cvat.apps.engine.log import slogger from cvat.apps.engine.media_extractors import (MEDIA_TYPES, Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, ValidateDimension, ZipChunkWriter, ZipCompressedChunkWriter, get_mime, sort) from cvat.apps.engine.utils import av_scan_paths, get_rq_job_meta +from cvat.utils.http import make_requests_session from utils.dataset_manifest import ImageManifestManager, VideoManifestManager, is_manifest from utils.dataset_manifest.core import VideoManifestValidator from utils.dataset_manifest.utils import detect_related_images @@ -343,24 +343,26 @@ def _validate_url(url): def _download_data(urls, upload_dir): job = rq.get_current_job() local_files = {} - for url in urls: - name = os.path.basename(urlrequest.url2pathname(urlparse.urlparse(url).path)) - if name in local_files: - raise Exception("filename collision: {}".format(name)) - _validate_url(url) - slogger.glob.info("Downloading: {}".format(url)) - job.meta['status'] = '{} is being downloaded..'.format(url) - job.save_meta() - response = requests.get(url, stream=True) - if response.status_code == 200: - response.raw.decode_content = True - with open(os.path.join(upload_dir, name), 'wb') as output_file: - shutil.copyfileobj(response.raw, output_file) - else: - raise Exception("Failed to download " + url) + with make_requests_session() as session: + for url in urls: + name = os.path.basename(urlrequest.url2pathname(urlparse.urlparse(url).path)) + if name in local_files: + raise Exception("filename collision: {}".format(name)) + _validate_url(url) + slogger.glob.info("Downloading: {}".format(url)) + job.meta['status'] = '{} is being downloaded..'.format(url) + job.save_meta() + + response = session.get(url, stream=True) + if response.status_code == 200: + response.raw.decode_content = True + with open(os.path.join(upload_dir, name), 'wb') as output_file: + shutil.copyfileobj(response.raw, output_file) + else: + raise Exception("Failed to download " + url) - local_files[name] = True + local_files[name] = True return list(local_files.keys()) diff --git a/cvat/apps/health/backends.py b/cvat/apps/health/backends.py index 3a8b1a058fab9827a380c37c7b225f02c512a727..df0e1f891d369755b913910199cc5295e68664e9 100644 --- a/cvat/apps/health/backends.py +++ b/cvat/apps/health/backends.py @@ -12,14 +12,17 @@ from health_check.exceptions import ServiceReturnedUnexpectedResult, ServiceUnav from django.conf import settings from django.core.cache import CacheKeyWarning, caches +from cvat.utils.http import make_requests_session + class OPAHealthCheck(BaseHealthCheckBackend): critical_service = True def check_status(self): opa_health_url = f'{settings.IAM_OPA_HOST}/health?bundles' try: - response = requests.get(opa_health_url) - response.raise_for_status() + with make_requests_session() as session: + response = session.get(opa_health_url) + response.raise_for_status() except requests.RequestException as e: raise HealthCheckException(str(e)) diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index c8e9e9d4d6426af77b00cd3bb699c5fa60d16326..df0727ccfd07ae3beeeb30f4c1abda4bdf981e6c 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -11,7 +11,6 @@ from collections import namedtuple from enum import Enum from typing import Any, List, Optional, Sequence, cast -import requests from attrs import define, field from django.conf import settings from django.db.models import Q @@ -21,6 +20,7 @@ from rest_framework.permissions import BasePermission from cvat.apps.organizations.models import Membership, Organization from cvat.apps.engine.models import Label, Project, Task, Job, Issue from cvat.apps.webhooks.models import WebhookTypeChoice +from cvat.utils.http import make_requests_session class StrEnum(str, Enum): @@ -107,8 +107,9 @@ class OpenPolicyAgentPermission(metaclass=ABCMeta): return None def check_access(self) -> PermissionResult: - response = requests.post(self.url, json=self.payload) - output = response.json()['result'] + with make_requests_session() as session: + response = session.post(self.url, json=self.payload) + output = response.json()['result'] allow = False reasons = [] @@ -124,14 +125,17 @@ class OpenPolicyAgentPermission(metaclass=ABCMeta): def filter(self, queryset): url = self.url.replace('/allow', '/filter') - r = requests.post(url, json=self.payload) + + with make_requests_session() as session: + r = session.post(url, json=self.payload).json()['result'] + q_objects = [] ops_dict = { '|': operator.or_, '&': operator.and_, '~': operator.not_, } - for item in r.json()['result']: + for item in r: if isinstance(item, str): val1 = q_objects.pop() if item == '~': @@ -384,9 +388,12 @@ class EventsPermission(OpenPolicyAgentPermission): def filter(self, query_params): url = self.url.replace('/allow', '/filter') - r = requests.post(url, json=self.payload) + + with make_requests_session() as session: + r = session.post(url, json=self.payload).json()['result'] + filter_params = query_params.copy() - for query in r.json()['result']: + for query in r: for attr, value in query.items(): if filter_params.get(attr, value) != value: raise PermissionDenied(f"You don't have permission to view events with {attr}={filter_params.get(attr)}") diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index a539751e780226ebb8138f79788d6f0847650de5..00bea72eb2e64472ceec144d3575602864a51e77 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -26,6 +26,7 @@ from cvat.apps.engine.frame_provider import FrameProvider from cvat.apps.engine.models import Job, Task from cvat.apps.engine.serializers import LabeledDataSerializer from cvat.apps.engine.models import ShapeType, SourceType +from cvat.utils.http import make_requests_session from drf_spectacular.utils import (extend_schema, extend_schema_view, OpenApiResponse, OpenApiParameter, inline_serializer) @@ -65,10 +66,11 @@ class LambdaGateway: else: url = NUCLIO_GATEWAY - reply = getattr(requests, method)(url, headers=extra_headers, - timeout=NUCLIO_TIMEOUT, json=data) - reply.raise_for_status() - response = reply.json() + with make_requests_session() as session: + reply = session.request(method, url, headers=extra_headers, + timeout=NUCLIO_TIMEOUT, json=data) + reply.raise_for_status() + response = reply.json() return response @@ -97,9 +99,11 @@ class LambdaGateway: url = f'http://host.docker.internal:{func.port}' else: url = f'http://localhost:{func.port}' - reply = requests.post(url, timeout=NUCLIO_TIMEOUT, json=payload) - reply.raise_for_status() - response = reply.json() + + with make_requests_session() as session: + reply = session.post(url, timeout=NUCLIO_TIMEOUT, json=payload) + reply.raise_for_status() + response = reply.json() return response diff --git a/cvat/apps/webhooks/signals.py b/cvat/apps/webhooks/signals.py index a5ff6b0db849503ad248619926efc6b27564bc92..5bc7444ac21927f453a262f1eae81ce4f55b4204 100644 --- a/cvat/apps/webhooks/signals.py +++ b/cvat/apps/webhooks/signals.py @@ -15,6 +15,7 @@ from django.conf import settings from cvat.apps.engine.models import Project from cvat.apps.engine.serializers import BasicUserSerializer from cvat.apps.organizations.models import Organization +from cvat.utils.http import make_requests_session from .event_type import EventTypeChoice, event_name from .models import Webhook, WebhookDelivery, WebhookTypeChoice @@ -43,16 +44,17 @@ def send_webhook(webhook, payload, delivery): response_body = None try: - response = requests.post( - webhook.target_url, - json=payload, - verify=webhook.enable_ssl, - headers=headers, - timeout=WEBHOOK_TIMEOUT, - stream=True, - ) - status_code = response.status_code - response_body = response.raw.read(RESPONSE_SIZE_LIMIT + 1, decode_content=True) + with make_requests_session() as session: + response = session.post( + webhook.target_url, + json=payload, + verify=webhook.enable_ssl, + headers=headers, + timeout=WEBHOOK_TIMEOUT, + stream=True, + ) + status_code = response.status_code + response_body = response.raw.read(RESPONSE_SIZE_LIMIT + 1, decode_content=True) except requests.ConnectionError: status_code = HTTPStatus.BAD_GATEWAY except requests.Timeout: diff --git a/cvat/utils/__init__.py b/cvat/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ff847f01203d01af18fd532b80eb646ca70bc663 --- /dev/null +++ b/cvat/utils/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/cvat/utils/http.py b/cvat/utils/http.py new file mode 100644 index 0000000000000000000000000000000000000000..c713871e1192598cdcb0dd60995870fbf8232f5c --- /dev/null +++ b/cvat/utils/http.py @@ -0,0 +1,15 @@ +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import requests +import requests.utils + +from cvat import __version__ + +_CVAT_USER_AGENT = f"CVAT/{__version__} {requests.utils.default_user_agent()}" + +def make_requests_session() -> requests.Session: + session = requests.Session() + session.headers['User-Agent'] = _CVAT_USER_AGENT + return session