diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e4c2e106017766589e18f3bdb89a3851c73fe2e..0a4a04e07b244822aa831799334844cc8f63e989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - Updated ELK to 6.8.23 which uses log4j 2.17.1 () +- Added validation for URLs which used as remote data source () ## \[1.7.0] - 2021-11-15 diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index dacb7f466f888cbb3c7ab936de13fef3a94b4b4d..cfea2ff743afa21d55d67e4403dba9b258a0b876 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -6202,9 +6202,9 @@ "deprecated": "Please see https://github.com/lydell/urix#deprecated" }, "node_modules/url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -11282,9 +11282,9 @@ "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" }, "url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "requires": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 89079929576e395965867f8477a9f7e87aa3d3b0..33464493de3136b19e782cf6e3c57ceb7b90c50f 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -6,6 +6,7 @@ import itertools import os import sys +from rest_framework.serializers import ValidationError import rq import re import shutil @@ -14,6 +15,8 @@ from traceback import print_exception from urllib import parse as urlparse from urllib import request as urlrequest import requests +import ipaddress +import dns.resolver import django_rq from django.conf import settings @@ -203,6 +206,45 @@ def _validate_manifest(manifests, root_dir): raise Exception('Invalid manifest was uploaded') return None +def _validate_url(url): + def _validate_ip_address(ip_address): + if not ip_address.is_global: + raise ValidationError('Non public IP address \'{}\' is provided!'.format(ip_address)) + + ALLOWED_SCHEMES = ['http', 'https'] + + parsed_url = urlparse.urlparse(url) + + if parsed_url.scheme not in ALLOWED_SCHEMES: + raise ValueError('Unsupported URL sheme: {}. Only http and https are supported'.format(parsed_url.scheme)) + + try: + ip_address = ipaddress.ip_address(parsed_url.hostname) + _validate_ip_address(ip_address) + except ValueError as _: + ip_v4_records = None + ip_v6_records = None + try: + ip_v4_records = dns.resolver.query(parsed_url.hostname, 'A') + for record in ip_v4_records: + _validate_ip_address(ipaddress.ip_address(record.to_text())) + except ValidationError: + raise + except Exception as e: + slogger.glob.info('Cannot get A record for domain \'{}\': {}'.format(parsed_url.hostname, e)) + + try: + ip_v6_records = dns.resolver.query(parsed_url.hostname, 'AAAA') + for record in ip_v6_records: + _validate_ip_address(ipaddress.ip_address(record.to_text())) + except ValidationError: + raise + except Exception as e: + slogger.glob.info('Cannot get AAAA record for domain \'{}\': {}'.format(parsed_url.hostname, e)) + + if not ip_v4_records and not ip_v6_records: + raise ValidationError('Cannot resolve IP address for domain \'{}\''.format(parsed_url.hostname)) + def _download_data(urls, upload_dir): job = rq.get_current_job() local_files = {} @@ -210,6 +252,7 @@ def _download_data(urls, upload_dir): name = os.path.basename(urlrequest.url2pathname(urlparse.urlparse(url).path)) if name in local_files: raise Exception("filename collision: {}".format(name)) + _validate_url(url) slogger.glob.info("Downloading: {}".format(url)) job.meta['status'] = '{} is being downloaded..'.format(url) job.save_meta() diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index d01f0f2c054b307bd176566bf198d988115bb4ae..e2a98b2ec5aeb4c7e92e4360541ff6bdef20163d 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -52,3 +52,4 @@ datumaro==0.2.0 --no-binary=datumaro urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability natsort==8.0.0 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability +dnspython==2.2.0 diff --git a/package-lock.json b/package-lock.json index 6bde3365e61c456183152ace47b765689936828e..d358f35577189f2815bc9c0d35a695772c0983ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -114,7 +114,7 @@ "devDependencies": {} }, "cvat-core": { - "version": "4.2.0", + "version": "4.2.1", "license": "MIT", "dependencies": { "axios": "^0.21.4", @@ -253,7 +253,7 @@ "devDependencies": {} }, "cvat-ui": { - "version": "1.35.2", + "version": "1.36.0", "license": "MIT", "dependencies": { "@ant-design/icons": "^4.6.3", @@ -292,12 +292,12 @@ "react-router": "^5.1.0", "react-router-dom": "^5.1.0", "react-share": "^4.4.0", + "react-sortable-hoc": "^2.0.0", "redux": "^4.1.1", "redux-devtools-extension": "^2.13.9", "redux-logger": "^3.0.6", "redux-thunk": "^2.3.0" - }, - "devDependencies": {} + } }, "node_modules/@ant-design/colors": { "version": "6.0.0", @@ -19388,6 +19388,21 @@ "react": "^16.3.0 || ^17" } }, + "node_modules/react-sortable-hoc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-2.0.0.tgz", + "integrity": "sha512-JZUw7hBsAHXK7PTyErJyI7SopSBFRcFHDjWW5SWjcugY0i6iH7f+eJkY8cJmGMlZ1C9xz1J3Vjz0plFpavVeRg==", + "dependencies": { + "@babel/runtime": "^7.2.0", + "invariant": "^2.2.4", + "prop-types": "^15.5.7" + }, + "peerDependencies": { + "prop-types": "^15.5.7", + "react": "^16.3.0 || ^17.0.0", + "react-dom": "^16.3.0 || ^17.0.0" + } + }, "node_modules/react-svg-core": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/react-svg-core/-/react-svg-core-3.0.3.tgz", @@ -24708,9 +24723,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -32439,6 +32454,7 @@ "react-router": "^5.1.0", "react-router-dom": "^5.1.0", "react-share": "^4.4.0", + "react-sortable-hoc": "^2.0.0", "redux": "^4.1.1", "redux-devtools-extension": "^2.13.9", "redux-logger": "^3.0.6", @@ -41752,6 +41768,16 @@ "jsonp": "^0.2.1" } }, + "react-sortable-hoc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-2.0.0.tgz", + "integrity": "sha512-JZUw7hBsAHXK7PTyErJyI7SopSBFRcFHDjWW5SWjcugY0i6iH7f+eJkY8cJmGMlZ1C9xz1J3Vjz0plFpavVeRg==", + "requires": { + "@babel/runtime": "^7.2.0", + "invariant": "^2.2.4", + "prop-types": "^15.5.7" + } + }, "react-svg-core": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/react-svg-core/-/react-svg-core-3.0.3.tgz", @@ -45971,9 +45997,9 @@ } }, "url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "requires": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" diff --git a/site/package-lock.json b/site/package-lock.json index 815d4598768ba72267b9cab710db92675572f85a..f8c33df1178439f693a3f93777ccae812ec4d55c 100644 --- a/site/package-lock.json +++ b/site/package-lock.json @@ -604,9 +604,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.1.28", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.28.tgz", - "integrity": "sha512-gSu9VZ2HtmoKYe/lmyPFES5nknFrHa+/DT9muUFWFMi6Jh9E1I7bkvlQ8xxf1Kos9pi9o8lBnIOkatMhKX/YUw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" @@ -1448,9 +1448,9 @@ "dev": true }, "nanoid": { - "version": "3.1.28", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.28.tgz", - "integrity": "sha512-gSu9VZ2HtmoKYe/lmyPFES5nknFrHa+/DT9muUFWFMi6Jh9E1I7bkvlQ8xxf1Kos9pi9o8lBnIOkatMhKX/YUw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", + "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true }, "node-releases": {