Merge remote-tracking branch 'origin/develop' into release-2.0.0

CHANGELOG.md

@@ -81,6 +81,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 - Updated ELK to 6.8.23 which uses log4j 2.17.1 (<https://github.com/openvinotoolkit/cvat/pull/4206>)
+- Added validation for URLs which used as remote data source (<https://github.com/openvinotoolkit/cvat/pull/4387>)
 
 ## \[1.7.0] - 2021-11-15
 

cvat-core/package-lock.json

@@ -6202,9 +6202,9 @@
       "deprecated": "Please see https://github.com/lydell/urix#deprecated"
     },
     "node_modules/url-parse": {
-      "version": "1.5.7",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz",
-      "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "dependencies": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"
@@ -11282,9 +11282,9 @@
       "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
     },
     "url-parse": {
-      "version": "1.5.7",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz",
-      "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "requires": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"

cvat/apps/engine/task.py

@@ -6,6 +6,7 @@
 import itertools
 import os
 import sys
+from rest_framework.serializers import ValidationError
 import rq
 import re
 import shutil
@@ -14,6 +15,8 @@ from traceback import print_exception
 from urllib import parse as urlparse
 from urllib import request as urlrequest
 import requests
+import ipaddress
+import dns.resolver
 import django_rq
 
 from django.conf import settings
@@ -203,6 +206,45 @@ def _validate_manifest(manifests, root_dir):
         raise Exception('Invalid manifest was uploaded')
     return None
 
+def _validate_url(url):
+    def _validate_ip_address(ip_address):
+        if not ip_address.is_global:
+            raise ValidationError('Non public IP address \'{}\' is provided!'.format(ip_address))
+
+    ALLOWED_SCHEMES = ['http', 'https']
+
+    parsed_url = urlparse.urlparse(url)
+
+    if parsed_url.scheme not in ALLOWED_SCHEMES:
+        raise ValueError('Unsupported URL sheme: {}. Only http and https are supported'.format(parsed_url.scheme))
+
+    try:
+        ip_address = ipaddress.ip_address(parsed_url.hostname)
+        _validate_ip_address(ip_address)
+    except ValueError as _:
+        ip_v4_records = None
+        ip_v6_records = None
+        try:
+            ip_v4_records = dns.resolver.query(parsed_url.hostname, 'A')
+            for record in ip_v4_records:
+                _validate_ip_address(ipaddress.ip_address(record.to_text()))
+        except ValidationError:
+            raise
+        except Exception as e:
+            slogger.glob.info('Cannot get A record for domain \'{}\': {}'.format(parsed_url.hostname, e))
+
+        try:
+            ip_v6_records = dns.resolver.query(parsed_url.hostname, 'AAAA')
+            for record in ip_v6_records:
+                _validate_ip_address(ipaddress.ip_address(record.to_text()))
+        except ValidationError:
+            raise
+        except Exception as e:
+            slogger.glob.info('Cannot get AAAA record for domain \'{}\': {}'.format(parsed_url.hostname, e))
+
+        if not ip_v4_records and not ip_v6_records:
+            raise ValidationError('Cannot resolve IP address for domain \'{}\''.format(parsed_url.hostname))
+
 def _download_data(urls, upload_dir):
     job = rq.get_current_job()
     local_files = {}
@@ -210,6 +252,7 @@ def _download_data(urls, upload_dir):
         name = os.path.basename(urlrequest.url2pathname(urlparse.urlparse(url).path))
         if name in local_files:
             raise Exception("filename collision: {}".format(name))
+        _validate_url(url)
         slogger.glob.info("Downloading: {}".format(url))
         job.meta['status'] = '{} is being downloaded..'.format(url)
         job.save_meta()

cvat/requirements/base.txt

@@ -52,3 +52,4 @@ datumaro==0.2.0 --no-binary=datumaro
 urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability
 natsort==8.0.0
 mistune>=2.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
+dnspython==2.2.0

package-lock.json

@@ -114,7 +114,7 @@
       "devDependencies": {}
     },
     "cvat-core": {
-      "version": "4.2.0",
+      "version": "4.2.1",
       "license": "MIT",
       "dependencies": {
         "axios": "^0.21.4",
@@ -253,7 +253,7 @@
       "devDependencies": {}
     },
     "cvat-ui": {
-      "version": "1.35.2",
+      "version": "1.36.0",
       "license": "MIT",
       "dependencies": {
         "@ant-design/icons": "^4.6.3",
@@ -292,12 +292,12 @@
         "react-router": "^5.1.0",
         "react-router-dom": "^5.1.0",
         "react-share": "^4.4.0",
+        "react-sortable-hoc": "^2.0.0",
         "redux": "^4.1.1",
         "redux-devtools-extension": "^2.13.9",
         "redux-logger": "^3.0.6",
         "redux-thunk": "^2.3.0"
-      },
-      "devDependencies": {}
+      }
     },
     "node_modules/@ant-design/colors": {
       "version": "6.0.0",
@@ -19388,6 +19388,21 @@
         "react": "^16.3.0 || ^17"
       }
     },
+    "node_modules/react-sortable-hoc": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-2.0.0.tgz",
+      "integrity": "sha512-JZUw7hBsAHXK7PTyErJyI7SopSBFRcFHDjWW5SWjcugY0i6iH7f+eJkY8cJmGMlZ1C9xz1J3Vjz0plFpavVeRg==",
+      "dependencies": {
+        "@babel/runtime": "^7.2.0",
+        "invariant": "^2.2.4",
+        "prop-types": "^15.5.7"
+      },
+      "peerDependencies": {
+        "prop-types": "^15.5.7",
+        "react": "^16.3.0 || ^17.0.0",
+        "react-dom": "^16.3.0 || ^17.0.0"
+      }
+    },
     "node_modules/react-svg-core": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/react-svg-core/-/react-svg-core-3.0.3.tgz",
@@ -24708,9 +24723,9 @@
       }
     },
     "node_modules/url-parse": {
-      "version": "1.5.7",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz",
-      "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "dependencies": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"
@@ -32439,6 +32454,7 @@
         "react-router": "^5.1.0",
         "react-router-dom": "^5.1.0",
         "react-share": "^4.4.0",
+        "react-sortable-hoc": "^2.0.0",
         "redux": "^4.1.1",
         "redux-devtools-extension": "^2.13.9",
         "redux-logger": "^3.0.6",
@@ -41752,6 +41768,16 @@
         "jsonp": "^0.2.1"
       }
     },
+    "react-sortable-hoc": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/react-sortable-hoc/-/react-sortable-hoc-2.0.0.tgz",
+      "integrity": "sha512-JZUw7hBsAHXK7PTyErJyI7SopSBFRcFHDjWW5SWjcugY0i6iH7f+eJkY8cJmGMlZ1C9xz1J3Vjz0plFpavVeRg==",
+      "requires": {
+        "@babel/runtime": "^7.2.0",
+        "invariant": "^2.2.4",
+        "prop-types": "^15.5.7"
+      }
+    },
     "react-svg-core": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/react-svg-core/-/react-svg-core-3.0.3.tgz",
@@ -45971,9 +45997,9 @@
       }
     },
     "url-parse": {
-      "version": "1.5.7",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz",
-      "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==",
+      "version": "1.5.10",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
+      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
       "requires": {
         "querystringify": "^2.1.1",
         "requires-port": "^1.0.0"

site/package-lock.json

@@ -604,9 +604,9 @@
       "dev": true
     },
     "node_modules/nanoid": {
-      "version": "3.1.28",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.28.tgz",
-      "integrity": "sha512-gSu9VZ2HtmoKYe/lmyPFES5nknFrHa+/DT9muUFWFMi6Jh9E1I7bkvlQ8xxf1Kos9pi9o8lBnIOkatMhKX/YUw==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz",
+      "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==",
       "dev": true,
       "bin": {
         "nanoid": "bin/nanoid.cjs"
@@ -1448,9 +1448,9 @@
       "dev": true
     },
     "nanoid": {
-      "version": "3.1.28",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.28.tgz",
-      "integrity": "sha512-gSu9VZ2HtmoKYe/lmyPFES5nknFrHa+/DT9muUFWFMi6Jh9E1I7bkvlQ8xxf1Kos9pi9o8lBnIOkatMhKX/YUw==",
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz",
+      "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==",
       "dev": true
     },
     "node-releases": {