Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
疯人忠
Cvat
提交
fd7d8024
C
Cvat
项目概览
疯人忠
/
Cvat
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
Cvat
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
fd7d8024
编写于
1月 06, 2023
作者:
A
Andrey Zhavoronkov
提交者:
GitHub
1月 06, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Splitted default queue to import/export, fixed django-rq admin page (#5555)
上级
0ea14d23
变更
25
隐藏空白更改
内联
并排
Showing
25 changed file
with
451 addition
and
144 deletion
+451
-144
.vscode/launch.json
.vscode/launch.json
+28
-6
components/serverless/docker-compose.serverless.yml
components/serverless/docker-compose.serverless.yml
+1
-1
cvat/apps/dataset_manager/views.py
cvat/apps/dataset_manager/views.py
+2
-1
cvat/apps/dataset_repo/dataset_repo.py
cvat/apps/dataset_repo/dataset_repo.py
+2
-1
cvat/apps/dataset_repo/views.py
cvat/apps/dataset_repo/views.py
+4
-3
cvat/apps/engine/backup.py
cvat/apps/engine/backup.py
+19
-14
cvat/apps/engine/mixins.py
cvat/apps/engine/mixins.py
+13
-4
cvat/apps/engine/task.py
cvat/apps/engine/task.py
+3
-3
cvat/apps/engine/views.py
cvat/apps/engine/views.py
+43
-22
cvat/apps/lambda_manager/views.py
cvat/apps/lambda_manager/views.py
+1
-2
cvat/apps/webhooks/signals.py
cvat/apps/webhooks/signals.py
+2
-1
cvat/settings/base.py
cvat/settings/base.py
+16
-3
docker-compose.dev.yml
docker-compose.dev.yml
+13
-5
docker-compose.yml
docker-compose.yml
+26
-7
helm-chart/Chart.yaml
helm-chart/Chart.yaml
+1
-1
helm-chart/templates/cvat_backend/worker_annotation/deployment.yml
...t/templates/cvat_backend/worker_annotation/deployment.yml
+170
-0
helm-chart/templates/cvat_backend/worker_export/deployment.yml
...chart/templates/cvat_backend/worker_export/deployment.yml
+21
-23
helm-chart/templates/cvat_backend/worker_import/deployment.yml
...chart/templates/cvat_backend/worker_import/deployment.yml
+22
-24
helm-chart/values.yaml
helm-chart/values.yaml
+12
-2
supervisord/all.conf
supervisord/all.conf
+11
-5
supervisord/worker.annotation.conf
supervisord/worker.annotation.conf
+2
-10
supervisord/worker.export.conf
supervisord/worker.export.conf
+32
-0
supervisord/worker.import.conf
supervisord/worker.import.conf
+5
-4
tests/docker-compose.file_share.yml
tests/docker-compose.file_share.yml
+1
-1
tests/values.test.yaml
tests/values.test.yaml
+1
-1
未找到文件。
.vscode/launch.json
浏览文件 @
fd7d8024
...
...
@@ -133,7 +133,7 @@
}
},
{
"name"
:
"server: RQ -
defaul
t"
,
"name"
:
"server: RQ -
impor
t"
,
"type"
:
"python"
,
"request"
:
"launch"
,
"stopOnEntry"
:
false
,
...
...
@@ -142,7 +142,26 @@
"program"
:
"${workspaceRoot}/manage.py"
,
"args"
:
[
"rqworker"
,
"default"
,
"import"
,
"--worker-class"
,
"cvat.rqworker.SimpleWorker"
,
],
"django"
:
true
,
"cwd"
:
"${workspaceFolder}"
,
"env"
:
{},
"console"
:
"internalConsole"
},
{
"name"
:
"server: RQ - export"
,
"type"
:
"python"
,
"request"
:
"launch"
,
"stopOnEntry"
:
false
,
"justMyCode"
:
false
,
"python"
:
"${command:python.interpreterPath}"
,
"program"
:
"${workspaceRoot}/manage.py"
,
"args"
:
[
"rqworker"
,
"export"
,
"--worker-class"
,
"cvat.rqworker.SimpleWorker"
,
],
...
...
@@ -161,6 +180,8 @@
"program"
:
"${workspaceRoot}/manage.py"
,
"args"
:
[
"rqscheduler"
,
"--queue"
,
"export"
],
"django"
:
true
,
"cwd"
:
"${workspaceFolder}"
,
...
...
@@ -168,7 +189,7 @@
"console"
:
"internalConsole"
},
{
"name"
:
"server: RQ -
low
"
,
"name"
:
"server: RQ -
annotation
"
,
"type"
:
"python"
,
"request"
:
"launch"
,
"justMyCode"
:
false
,
...
...
@@ -177,7 +198,7 @@
"program"
:
"${workspaceRoot}/manage.py"
,
"args"
:
[
"rqworker"
,
"
low
"
,
"
annotation
"
,
"--worker-class"
,
"cvat.rqworker.SimpleWorker"
,
],
...
...
@@ -379,8 +400,9 @@
"name"
:
"server: debug"
,
"configurations"
:
[
"server: django"
,
"server: RQ - default"
,
"server: RQ - low"
,
"server: RQ - import"
,
"server: RQ - export"
,
"server: RQ - annotation"
,
"server: RQ - webhooks"
,
"server: RQ - scheduler"
,
"server: git"
,
...
...
components/serverless/docker-compose.serverless.yml
浏览文件 @
fd7d8024
...
...
@@ -28,6 +28,6 @@ services:
extra_hosts
:
-
"
host.docker.internal:host-gateway"
cvat_worker_
low
:
cvat_worker_
annotation
:
extra_hosts
:
-
"
host.docker.internal:host-gateway"
cvat/apps/dataset_manager/views.py
浏览文件 @
fd7d8024
...
...
@@ -11,6 +11,7 @@ import django_rq
from
datumaro.util.os_util
import
make_file_name
from
datumaro.util
import
to_snake_case
from
django.utils
import
timezone
from
django.conf
import
settings
import
cvat.apps.dataset_manager.task
as
task
import
cvat.apps.dataset_manager.project
as
project
...
...
@@ -83,7 +84,7 @@ def export(dst_format, project_id=None, task_id=None, job_id=None, server_url=No
os
.
replace
(
temp_file
,
output_path
)
archive_ctime
=
osp
.
getctime
(
output_path
)
scheduler
=
django_rq
.
get_scheduler
()
scheduler
=
django_rq
.
get_scheduler
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
cleaning_job
=
scheduler
.
enqueue_in
(
time_delta
=
cache_ttl
,
func
=
clear_export_cache
,
file_path
=
output_path
,
...
...
cvat/apps/dataset_repo/dataset_repo.py
浏览文件 @
fd7d8024
...
...
@@ -15,6 +15,7 @@ import django_rq
import
git
from
django.db
import
transaction
from
django.utils
import
timezone
from
django.conf
import
settings
from
cvat.apps.dataset_manager.formats.registry
import
format_for
from
cvat.apps.dataset_manager.task
import
export_task
...
...
@@ -427,7 +428,7 @@ def get(tid, user):
response
[
'url'
][
'value'
]
=
'{} [{}]'
.
format
(
db_git
.
url
,
db_git
.
path
)
try
:
rq_id
=
"git.push.{}"
.
format
(
tid
)
queue
=
django_rq
.
get_queue
(
'default'
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
rq_job
is
not
None
and
(
rq_job
.
is_queued
or
rq_job
.
is_started
):
db_git
.
status
=
GitStatusChoice
.
SYNCING
...
...
cvat/apps/dataset_repo/views.py
浏览文件 @
fd7d8024
...
...
@@ -5,6 +5,7 @@ import http.client
from
django.http
import
HttpResponseBadRequest
,
HttpResponse
from
rules.contrib.views
import
permission_required
,
objectgetter
from
django.conf
import
settings
from
rest_framework.permissions
import
IsAuthenticated
from
rest_framework.response
import
Response
...
...
@@ -37,7 +38,7 @@ def _legacy_api_view(allowed_method_names=None):
@
_legacy_api_view
()
def
check_process
(
request
,
rq_id
):
try
:
queue
=
django_rq
.
get_queue
(
'default'
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
rq_job
is
not
None
:
...
...
@@ -65,7 +66,7 @@ def create(request: Request, tid):
export_format
=
body
.
get
(
"format"
)
lfs
=
body
[
"lfs"
]
rq_id
=
"git.create.{}"
.
format
(
tid
)
queue
=
django_rq
.
get_queue
(
"default"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
queue
.
enqueue_call
(
func
=
CVATGit
.
initial_create
,
args
=
(
tid
,
path
,
export_format
,
lfs
,
request
.
user
),
job_id
=
rq_id
)
return
Response
({
"rq_id"
:
rq_id
})
...
...
@@ -80,7 +81,7 @@ def push_repository(request: Request, tid):
slogger
.
task
[
tid
].
info
(
"push repository request"
)
rq_id
=
"git.push.{}"
.
format
(
tid
)
queue
=
django_rq
.
get_queue
(
'default'
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
queue
.
enqueue_call
(
func
=
CVATGit
.
push
,
args
=
(
tid
,
request
.
user
,
request
.
scheme
,
request
.
get_host
()),
job_id
=
rq_id
)
return
Response
({
"rq_id"
:
rq_id
})
...
...
cvat/apps/engine/backup.py
浏览文件 @
fd7d8024
...
...
@@ -712,7 +712,7 @@ def _create_backup(db_instance, Exporter, output_path, logger, cache_ttl):
os
.
replace
(
temp_file
,
output_path
)
archive_ctime
=
os
.
path
.
getctime
(
output_path
)
scheduler
=
django_rq
.
get_scheduler
()
scheduler
=
django_rq
.
get_scheduler
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
cleaning_job
=
scheduler
.
enqueue_in
(
time_delta
=
cache_ttl
,
func
=
clear_export_cache
,
file_path
=
output_path
,
...
...
@@ -731,7 +731,7 @@ def _create_backup(db_instance, Exporter, output_path, logger, cache_ttl):
log_exception
(
logger
)
raise
def
export
(
db_instance
,
request
):
def
export
(
db_instance
,
request
,
queue_name
):
action
=
request
.
query_params
.
get
(
'action'
,
None
)
filename
=
request
.
query_params
.
get
(
'filename'
,
None
)
...
...
@@ -740,13 +740,13 @@ def export(db_instance, request):
"Unexpected action specified for the request"
)
if
isinstance
(
db_instance
,
Task
):
filename_prefix
=
'task'
obj_type
=
'task'
logger
=
slogger
.
task
[
db_instance
.
pk
]
Exporter
=
TaskExporter
cache_ttl
=
TASK_CACHE_TTL
use_target_storage_conf
=
request
.
query_params
.
get
(
'use_default_location'
,
True
)
elif
isinstance
(
db_instance
,
Project
):
filename_prefix
=
'project'
obj_type
=
'project'
logger
=
slogger
.
project
[
db_instance
.
pk
]
Exporter
=
ProjectExporter
cache_ttl
=
PROJECT_CACHE_TTL
...
...
@@ -762,8 +762,8 @@ def export(db_instance, request):
field_name
=
StorageType
.
TARGET
)
queue
=
django_rq
.
get_queue
(
"default"
)
rq_id
=
"/api/{}s/{}/backup"
.
format
(
filename_prefix
,
db_instance
.
pk
)
queue
=
django_rq
.
get_queue
(
queue_name
)
rq_id
=
f
"export:
{
obj_type
}
.id
{
db_instance
.
pk
}
-by-
{
request
.
user
}
"
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
rq_job
:
last_project_update_time
=
timezone
.
localtime
(
db_instance
.
updated_date
)
...
...
@@ -780,7 +780,7 @@ def export(db_instance, request):
timestamp
=
datetime
.
strftime
(
last_project_update_time
,
"%Y_%m_%d_%H_%M_%S"
)
filename
=
filename
or
"{}_{}_backup_{}{}"
.
format
(
filename_prefix
,
db_instance
.
name
,
timestamp
,
obj_type
,
db_instance
.
name
,
timestamp
,
os
.
path
.
splitext
(
file_path
)[
1
]).
lower
()
location
=
location_conf
.
get
(
'location'
)
...
...
@@ -820,7 +820,7 @@ def export(db_instance, request):
ttl
=
dm
.
views
.
PROJECT_CACHE_TTL
.
total_seconds
()
queue
.
enqueue_call
(
func
=
_create_backup
,
args
=
(
db_instance
,
Exporter
,
'{}_backup.zip'
.
format
(
filename_prefix
),
logger
,
cache_ttl
),
args
=
(
db_instance
,
Exporter
,
'{}_backup.zip'
.
format
(
obj_type
),
logger
,
cache_ttl
),
job_id
=
rq_id
,
meta
=
{
'request_time'
:
timezone
.
localtime
()
},
result_ttl
=
ttl
,
failure_ttl
=
ttl
)
...
...
@@ -834,8 +834,7 @@ def _download_file_from_bucket(db_storage, filename, key):
with
open
(
filename
,
'wb+'
)
as
f
:
f
.
write
(
data
.
getbuffer
())
def
_import
(
importer
,
request
,
rq_id
,
Serializer
,
file_field_name
,
location_conf
,
filename
=
None
):
queue
=
django_rq
.
get_queue
(
"default"
)
def
_import
(
importer
,
request
,
queue
,
rq_id
,
Serializer
,
file_field_name
,
location_conf
,
filename
=
None
):
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
not
rq_job
:
...
...
@@ -905,11 +904,11 @@ def _import(importer, request, rq_id, Serializer, file_field_name, location_conf
def
get_backup_dirname
():
return
settings
.
TMP_FILES_ROOT
def
import_project
(
request
,
filename
=
None
):
def
import_project
(
request
,
queue_name
,
filename
=
None
):
if
'rq_id'
in
request
.
data
:
rq_id
=
request
.
data
[
'rq_id'
]
else
:
rq_id
=
"{}@/api/projects/{}/import"
.
format
(
request
.
user
,
uuid
.
uuid4
())
rq_id
=
f
"import:project.
{
uuid
.
uuid4
()
}
-by-
{
request
.
user
}
"
Serializer
=
ProjectFileSerializer
file_field_name
=
'project_file'
...
...
@@ -918,9 +917,12 @@ def import_project(request, filename=None):
field_name
=
StorageType
.
SOURCE
,
)
queue
=
django_rq
.
get_queue
(
queue_name
)
return
_import
(
importer
=
_import_project
,
request
=
request
,
queue
=
queue
,
rq_id
=
rq_id
,
Serializer
=
Serializer
,
file_field_name
=
file_field_name
,
...
...
@@ -928,11 +930,11 @@ def import_project(request, filename=None):
filename
=
filename
)
def
import_task
(
request
,
filename
=
None
):
def
import_task
(
request
,
queue_name
,
filename
=
None
):
if
'rq_id'
in
request
.
data
:
rq_id
=
request
.
data
[
'rq_id'
]
else
:
rq_id
=
"{}@/api/tasks/{}/import"
.
format
(
request
.
user
,
uuid
.
uuid4
())
rq_id
=
f
"import:task.
{
uuid
.
uuid4
()
}
-by-
{
request
.
user
}
"
Serializer
=
TaskFileSerializer
file_field_name
=
'task_file'
...
...
@@ -941,9 +943,12 @@ def import_task(request, filename=None):
field_name
=
StorageType
.
SOURCE
)
queue
=
django_rq
.
get_queue
(
queue_name
)
return
_import
(
importer
=
_import_task
,
request
=
request
,
queue
=
queue
,
rq_id
=
rq_id
,
Serializer
=
Serializer
,
file_field_name
=
file_field_name
,
...
...
cvat/apps/engine/mixins.py
浏览文件 @
fd7d8024
...
...
@@ -246,7 +246,7 @@ class UploadMixin:
class
AnnotationMixin
:
def
export_annotations
(
self
,
request
,
pk
,
db_obj
,
export_func
,
callback
,
get_data
=
None
):
format_name
=
request
.
query_params
.
get
(
"format"
)
format_name
=
request
.
query_params
.
get
(
"format"
,
""
)
action
=
request
.
query_params
.
get
(
"action"
,
""
).
lower
()
filename
=
request
.
query_params
.
get
(
"filename"
,
""
)
...
...
@@ -259,7 +259,8 @@ class AnnotationMixin:
field_name
=
StorageType
.
TARGET
,
)
rq_id
=
"/api/{}/{}/annotations/{}"
.
format
(
self
.
_object
.
__class__
.
__name__
.
lower
(),
pk
,
format_name
)
object_name
=
self
.
_object
.
__class__
.
__name__
.
lower
()
rq_id
=
f
"export:annotations-for-
{
object_name
}
.id
{
pk
}
-in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-format"
if
format_name
:
return
export_func
(
db_instance
=
self
.
_object
,
...
...
@@ -316,13 +317,21 @@ class AnnotationMixin:
class
SerializeMixin
:
def
serialize
(
self
,
request
,
export_func
):
db_object
=
self
.
get_object
()
# force to call check_object_permissions
return
export_func
(
db_object
,
request
)
return
export_func
(
db_object
,
request
,
queue_name
=
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
,
)
def
deserialize
(
self
,
request
,
import_func
):
location
=
request
.
query_params
.
get
(
"location"
,
Location
.
LOCAL
)
if
location
==
Location
.
CLOUD_STORAGE
:
file_name
=
request
.
query_params
.
get
(
"filename"
,
""
)
return
import_func
(
request
,
filename
=
file_name
)
return
import_func
(
request
,
queue_name
=
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
,
filename
=
file_name
,
)
return
self
.
upload_data
(
request
)
...
...
cvat/apps/engine/task.py
浏览文件 @
fd7d8024
...
...
@@ -38,11 +38,11 @@ from .cloud_provider import db_storage_to_storage_instance
############################# Low Level server API
def
create
(
tid
,
data
):
def
create
(
tid
,
data
,
username
):
"""Schedule the task"""
q
=
django_rq
.
get_queue
(
'default'
)
q
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
q
.
enqueue_call
(
func
=
_create_thread
,
args
=
(
tid
,
data
),
job_id
=
"/api/tasks/{}"
.
format
(
tid
)
)
job_id
=
f
"create:task.id
{
tid
}
-by-
{
username
}
"
)
@
transaction
.
atomic
def
rq_handler
(
job
,
exc_type
,
exc_value
,
traceback
):
...
...
cvat/apps/engine/views.py
浏览文件 @
fd7d8024
...
...
@@ -407,7 +407,7 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
resource_type_field_name
=
None
),
responses
=
{
'202'
:
OpenApiResponse
(
description
=
'
Ex
porting has been started'
),
'202'
:
OpenApiResponse
(
description
=
'
Im
porting has been started'
),
'400'
:
OpenApiResponse
(
description
=
'Failed to import dataset'
),
'405'
:
OpenApiResponse
(
description
=
'Format is not available'
),
})
...
...
@@ -415,6 +415,7 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
url_path
=
r
'dataset/?$'
)
def
dataset
(
self
,
request
,
pk
):
self
.
_object
=
self
.
get_object
()
# force to call check_object_permissions
rq_id
=
f
"import:dataset-for-porject.id
{
pk
}
-by-
{
request
.
user
}
"
if
request
.
method
in
{
'POST'
,
'OPTIONS'
}:
return
self
.
import_annotations
(
...
...
@@ -423,13 +424,13 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
db_obj
=
self
.
_object
,
import_func
=
_import_project_dataset
,
rq_func
=
dm
.
project
.
import_dataset_as_project
,
rq_id
=
f
"/api/project/
{
pk
}
/dataset_import"
,
rq_id
=
rq_id
,
)
else
:
action
=
request
.
query_params
.
get
(
"action"
,
""
).
lower
()
if
action
in
(
"import_status"
,):
queue
=
django_rq
.
get_queue
(
"default"
)
rq_job
=
queue
.
fetch_job
(
f
"/api/project/
{
pk
}
/dataset_import"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
rq_job
is
None
:
return
Response
(
status
=
status
.
HTTP_404_NOT_FOUND
)
elif
rq_job
.
is_finished
:
...
...
@@ -449,7 +450,10 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
)
else
:
return
Response
(
data
=
self
.
_get_rq_response
(
'default'
,
f
'/api/project/
{
pk
}
/dataset_import'
),
data
=
self
.
_get_rq_response
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
,
rq_id
,
),
status
=
status
.
HTTP_202_ACCEPTED
)
else
:
...
...
@@ -495,7 +499,7 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
return
_import_project_dataset
(
request
=
request
,
filename
=
uploaded_file
,
rq_id
=
f
"
/api/project/
{
self
.
_object
.
pk
}
/dataset_import
"
,
rq_id
=
f
"
import:dataset-for-porject.id
{
self
.
_object
.
pk
}
-by-
{
request
.
user
}
"
,
rq_func
=
dm
.
project
.
import_dataset_as_project
,
pk
=
self
.
_object
.
pk
,
format_name
=
format_name
,
...
...
@@ -507,10 +511,14 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
tmp_dir
=
backup
.
get_backup_dirname
()
backup_file
=
os
.
path
.
join
(
tmp_dir
,
filename
)
if
os
.
path
.
isfile
(
backup_file
):
return
backup
.
import_project
(
request
,
filename
=
backup_file
)
return
backup
.
import_project
(
request
,
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
,
filename
=
backup_file
,
)
return
Response
(
data
=
'No such file were uploaded'
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
return
backup
.
import_project
(
request
)
return
backup
.
import_project
(
request
,
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
return
Response
(
data
=
'Unknown upload was finished'
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
...
...
@@ -933,7 +941,8 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
return
_import_annotations
(
request
=
request
,
filename
=
annotation_file
,
rq_id
=
"{}@/api/tasks/{}/annotations/upload"
.
format
(
request
.
user
,
self
.
_object
.
pk
),
rq_id
=
(
f
"import:annotations-for-task.id
{
self
.
_object
.
pk
}
-"
f
"in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
),
rq_func
=
dm
.
task
.
import_task_annotations
,
pk
=
self
.
_object
.
pk
,
format_name
=
format_name
,
...
...
@@ -972,7 +981,7 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
# the value specified by the user or it's default value from the database
if
'stop_frame'
not
in
serializer
.
validated_data
:
data
[
'stop_frame'
]
=
None
task
.
create
(
self
.
_object
.
id
,
data
)
task
.
create
(
self
.
_object
.
id
,
data
,
request
.
user
)
return
Response
(
serializer
.
data
,
status
=
status
.
HTTP_202_ACCEPTED
)
elif
self
.
action
==
'import_backup'
:
filename
=
request
.
query_params
.
get
(
"filename"
,
""
)
...
...
@@ -980,10 +989,14 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
tmp_dir
=
backup
.
get_backup_dirname
()
backup_file
=
os
.
path
.
join
(
tmp_dir
,
filename
)
if
os
.
path
.
isfile
(
backup_file
):
return
backup
.
import_task
(
request
,
filename
=
backup_file
)
return
backup
.
import_task
(
request
,
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
,
filename
=
backup_file
,
)
return
Response
(
data
=
'No such file were uploaded'
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
return
backup
.
import_task
(
request
)
return
backup
.
import_task
(
request
,
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
return
Response
(
data
=
'Unknown upload was finished'
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
...
...
@@ -1155,16 +1168,17 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
return
Response
(
data
=
"Exporting annotations from a task without data is not allowed"
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
elif
request
.
method
==
'POST'
or
request
.
method
==
'OPTIONS'
:
format_name
=
request
.
query_params
.
get
(
'format'
,
''
)
return
self
.
import_annotations
(
request
=
request
,
pk
=
pk
,
db_obj
=
self
.
_object
,
import_func
=
_import_annotations
,
rq_func
=
dm
.
task
.
import_task_annotations
,
rq_id
=
"{}@/api/tasks/{}/annotations/upload"
.
format
(
request
.
user
,
pk
)
rq_id
=
f
"import:annotations-for-task.id
{
pk
}
-in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
)
elif
request
.
method
==
'PUT'
:
format_name
=
request
.
query_params
.
get
(
'format'
)
format_name
=
request
.
query_params
.
get
(
'format'
,
''
)
if
format_name
:
use_settings
=
strtobool
(
str
(
request
.
query_params
.
get
(
'use_default_location'
,
True
)))
conv_mask_to_poly
=
strtobool
(
request
.
query_params
.
get
(
'conv_mask_to_poly'
,
'True'
))
...
...
@@ -1174,7 +1188,7 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
)
return
_import_annotations
(
request
=
request
,
rq_id
=
"{}@/api/tasks/{}/annotations/upload"
.
format
(
request
.
user
,
pk
)
,
rq_id
=
f
"import:annotations-for-task.id
{
pk
}
-in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
,
rq_func
=
dm
.
task
.
import_task_annotations
,
pk
=
pk
,
format_name
=
format_name
,
...
...
@@ -1226,7 +1240,10 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
@
action
(
detail
=
True
,
methods
=
[
'GET'
],
serializer_class
=
RqStatusSerializer
)
def
status
(
self
,
request
,
pk
):
self
.
get_object
()
# force to call check_object_permissions
response
=
self
.
_get_rq_response
(
queue
=
"default"
,
job_id
=
f
"/api/tasks/
{
pk
}
"
)
response
=
self
.
_get_rq_response
(
queue
=
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
,
job_id
=
f
"create:task.id
{
pk
}
-by-
{
request
.
user
}
"
)
serializer
=
RqStatusSerializer
(
data
=
response
)
if
serializer
.
is_valid
(
raise_exception
=
True
):
...
...
@@ -1437,7 +1454,8 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
return
_import_annotations
(
request
=
request
,
filename
=
annotation_file
,
rq_id
=
"{}@/api/jobs/{}/annotations/upload"
.
format
(
request
.
user
,
self
.
_object
.
pk
),
rq_id
=
(
f
"import:annotations-for-job.id
{
self
.
_object
.
pk
}
-"
f
"in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
),
rq_func
=
dm
.
task
.
import_job_annotations
,
pk
=
self
.
_object
.
pk
,
format_name
=
format_name
,
...
...
@@ -1544,13 +1562,15 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
)
elif
request
.
method
==
'POST'
or
request
.
method
==
'OPTIONS'
:
format_name
=
request
.
query_params
.
get
(
'format'
,
''
)
return
self
.
import_annotations
(
request
=
request
,
pk
=
pk
,
db_obj
=
self
.
_object
.
segment
.
task
,
import_func
=
_import_annotations
,
rq_func
=
dm
.
task
.
import_job_annotations
,
rq_id
=
"{}@/api/jobs/{}/annotations/upload"
.
format
(
request
.
user
,
pk
)
rq_id
=
(
f
"import:annotations-for-job.id
{
self
.
_object
.
pk
}
-"
f
"in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
),
)
elif
request
.
method
==
'PUT'
:
...
...
@@ -1564,7 +1584,8 @@ class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
)
return
_import_annotations
(
request
=
request
,
rq_id
=
"{}@/api/jobs/{}/annotations/upload"
.
format
(
request
.
user
,
pk
),
rq_id
=
(
f
"import:annotations-for-job.id
{
pk
}
-"
f
"in-
{
format_name
.
replace
(
' '
,
'_'
)
}
-by-
{
request
.
user
}
"
),
rq_func
=
dm
.
task
.
import_job_annotations
,
pk
=
pk
,
format_name
=
format_name
,
...
...
@@ -2248,7 +2269,7 @@ def _import_annotations(request, rq_id, rq_func, pk, format_name,
elif
not
format_desc
.
ENABLED
:
return
Response
(
status
=
status
.
HTTP_405_METHOD_NOT_ALLOWED
)
queue
=
django_rq
.
get_queue
(
"default"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
not
rq_job
:
...
...
@@ -2329,7 +2350,7 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba
elif
not
format_desc
.
ENABLED
:
return
Response
(
status
=
status
.
HTTP_405_METHOD_NOT_ALLOWED
)
queue
=
django_rq
.
get_queue
(
"default"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
EXPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
rq_job
:
...
...
@@ -2420,7 +2441,7 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name, filename=N
elif
not
format_desc
.
ENABLED
:
return
Response
(
status
=
status
.
HTTP_405_METHOD_NOT_ALLOWED
)
queue
=
django_rq
.
get_queue
(
"default"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
IMPORT_DATA
.
value
)
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
not
rq_job
:
...
...
cvat/apps/lambda_manager/views.py
浏览文件 @
fd7d8024
...
...
@@ -364,8 +364,7 @@ class LambdaFunction:
class
LambdaQueue
:
def
_get_queue
(
self
):
QUEUE_NAME
=
"low"
return
django_rq
.
get_queue
(
QUEUE_NAME
)
return
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
AUTO_ANNOTATION
.
value
)
def
get_jobs
(
self
):
queue
=
self
.
_get_queue
()
...
...
cvat/apps/webhooks/signals.py
浏览文件 @
fd7d8024
...
...
@@ -10,6 +10,7 @@ import json
import
django_rq
import
requests
from
django.dispatch
import
Signal
,
receiver
from
django.conf
import
settings
from
cvat.apps.engine.models
import
Project
from
cvat.apps.engine.serializers
import
BasicUserSerializer
...
...
@@ -75,7 +76,7 @@ def add_to_queue(webhook, payload, redelivery=False):
response
=
""
,
)
queue
=
django_rq
.
get_queue
(
"webhooks"
)
queue
=
django_rq
.
get_queue
(
settings
.
CVAT_QUEUES
.
WEBHOOKS
.
value
)
queue
.
enqueue_call
(
func
=
send_webhook
,
args
=
(
webhook
,
payload
,
delivery
))
return
delivery
...
...
cvat/settings/base.py
浏览文件 @
fd7d8024
...
...
@@ -15,6 +15,7 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.0/ref/settings/
"""
from
enum
import
Enum
import
os
import
sys
import
fcntl
...
...
@@ -296,20 +297,32 @@ OLD_PASSWORD_FIELD_ENABLED = True
# Django-RQ
# https://github.com/rq/django-rq
class
CVAT_QUEUES
(
Enum
):
IMPORT_DATA
=
'import'
EXPORT_DATA
=
'export'
AUTO_ANNOTATION
=
'annotation'
WEBHOOKS
=
'webhooks'
RQ_QUEUES
=
{
'default'
:
{
CVAT_QUEUES
.
IMPORT_DATA
.
value
:
{
'HOST'
:
'localhost'
,
'PORT'
:
6379
,
'DB'
:
0
,
'DEFAULT_TIMEOUT'
:
'4h'
},
CVAT_QUEUES
.
EXPORT_DATA
.
value
:
{
'HOST'
:
'localhost'
,
'PORT'
:
6379
,
'DB'
:
0
,
'DEFAULT_TIMEOUT'
:
'4h'
},
'low'
:
{
CVAT_QUEUES
.
AUTO_ANNOTATION
.
value
:
{
'HOST'
:
'localhost'
,
'PORT'
:
6379
,
'DB'
:
0
,
'DEFAULT_TIMEOUT'
:
'24h'
},
'webhooks'
:
{
CVAT_QUEUES
.
WEBHOOKS
.
value
:
{
'HOST'
:
'localhost'
,
'PORT'
:
6379
,
'DB'
:
0
,
...
...
docker-compose.dev.yml
浏览文件 @
fd7d8024
...
...
@@ -16,7 +16,6 @@ services:
CLAM_AV
:
INSTALL_SOURCES
:
CVAT_DEBUG_ENABLED
:
command
:
'
-c
supervisord/server.conf'
environment
:
CVAT_DEBUG_ENABLED
:
'
${CVAT_DEBUG_ENABLED:-no}'
CVAT_DEBUG_PORT
:
'
9090'
...
...
@@ -25,8 +24,7 @@ services:
ports
:
-
'
9090:9090'
cvat_worker_default
:
command
:
-c supervisord/worker.default.conf
cvat_worker_export
:
environment
:
# For debugging, make sure to set 1 process
# Due to the supervisord specifics, the extra processes will fail and
...
...
@@ -37,8 +35,18 @@ services:
ports
:
-
'
9092:9092'
cvat_worker_low
:
command
:
-c supervisord/worker.low.conf
cvat_worker_import
:
environment
:
# For debugging, make sure to set 1 process
# Due to the supervisord specifics, the extra processes will fail and
# after few attempts supervisord will give up restarting, leaving only 1 process
# NUMPROCS: 1
CVAT_DEBUG_ENABLED
:
'
${CVAT_DEBUG_ENABLED:-no}'
CVAT_DEBUG_PORT
:
'
9093'
ports
:
-
'
9093:9093'
cvat_worker_annotation
:
environment
:
# For debugging, make sure to set 1 process
# Due to the supervisord specifics, the extra processes will fail and
...
...
docker-compose.yml
浏览文件 @
fd7d8024
...
...
@@ -82,20 +82,39 @@ services:
networks
:
-
cvat
cvat_worker_default
:
container_name
:
cvat_worker_default
cvat_worker_import
:
container_name
:
cvat_worker_import
image
:
cvat/server:${CVAT_VERSION:-dev}
restart
:
always
depends_on
:
-
cvat_redis
-
cvat_db
environment
:
CVAT_REDIS_HOST
:
'
cvat_redis'
CVAT_POSTGRES_HOST
:
'
cvat_db'
no_proxy
:
elasticsearch,kibana,logstash,nuclio,opa,${no_proxy:-}
NUMPROCS
:
2
command
:
-c supervisord/worker.import.conf
volumes
:
-
cvat_data:/home/django/data
-
cvat_keys:/home/django/keys
-
cvat_logs:/home/django/logs
networks
:
-
cvat
cvat_worker_export
:
container_name
:
cvat_worker_export
image
:
cvat/server:${CVAT_VERSION:-dev}
restart
:
always
depends_on
:
-
cvat_redis
-
cvat_db
-
cvat_opa
environment
:
CVAT_REDIS_HOST
:
'
cvat_redis'
CVAT_POSTGRES_HOST
:
'
cvat_db'
no_proxy
:
elasticsearch,kibana,logstash,nuclio,opa,${no_proxy:-}
NUMPROCS
:
2
command
:
-c supervisord/worker.
defaul
t.conf
command
:
-c supervisord/worker.
expor
t.conf
volumes
:
-
cvat_data:/home/django/data
-
cvat_keys:/home/django/keys
...
...
@@ -103,8 +122,8 @@ services:
networks
:
-
cvat
cvat_worker_
low
:
container_name
:
cvat_worker_
low
cvat_worker_
annotation
:
container_name
:
cvat_worker_
annotation
image
:
cvat/server:${CVAT_VERSION:-dev}
restart
:
always
depends_on
:
...
...
@@ -116,7 +135,7 @@ services:
CVAT_POSTGRES_HOST
:
'
cvat_db'
no_proxy
:
elasticsearch,kibana,logstash,nuclio,opa,${no_proxy:-}
NUMPROCS
:
1
command
:
-c supervisord/worker.
low
.conf
command
:
-c supervisord/worker.
annotation
.conf
volumes
:
-
cvat_data:/home/django/data
-
cvat_keys:/home/django/keys
...
...
helm-chart/Chart.yaml
浏览文件 @
fd7d8024
...
...
@@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version
:
0.
5.2
version
:
0.
6.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
...
...
helm-chart/templates/cvat_backend/worker_annotation/deployment.yml
0 → 100644
浏览文件 @
fd7d8024
apiVersion
:
apps/v1
kind
:
Deployment
metadata
:
name
:
{{
.Release.Name
}}
-backend-worker-annotation
namespace
:
{{
.Release.Namespace
}}
labels
:
app
:
cvat-app
tier
:
backend
component
:
worker-annotation
{{
- include "cvat.labels" . | nindent 4
}}
{{
- with .Values.cvat.backend.worker.annotation.labels
}}
{{
- toYaml . | nindent 4
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.annotations
}}
annotations
:
{{
- toYaml . | nindent 4
}}
{{
- end
}}
spec
:
replicas
:
{{
.Values.cvat.backend.worker.annotation.replicas
}}
strategy
:
type
:
Recreate
selector
:
matchLabels
:
{{
- include "cvat.labels" . | nindent 6
}}
{{
- with .Values.cvat.backend.worker.annotation.labels
}}
{{
- toYaml . | nindent 6
}}
{{
- end
}}
app
:
cvat-app
tier
:
backend
component
:
worker-annotation
template
:
metadata
:
labels
:
app
:
cvat-app
tier
:
backend
component
:
worker-annotation
{{
- include "cvat.labels" . | nindent 8
}}
{{
- with .Values.cvat.backend.worker.annotation.labels
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.annotations
}}
annotations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
spec
:
containers
:
-
name
:
cvat-app-backend-worker-annotation-container
image
:
{{
.Values.cvat.backend.image
}}
:{{ .Values.cvat.backend.tag }}
imagePullPolicy
:
{{
.Values.cvat.backend.imagePullPolicy
}}
{{
- with .Values.cvat.backend.worker.annotation.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
args
:
[
"
-c"
,
"
supervisord/worker.annotation.conf"
]
env
:
{{
- if .Values.redis.enabled
}}
-
name
:
CVAT_REDIS_HOST
value
:
"
{{
.Release.Name
}}-redis-master"
{{
- else
}}
-
name
:
CVAT_REDIS_HOST
value
:
"
{{
.Values.redis.external.host
}}"
{{
- end
}}
-
name
:
CVAT_REDIS_PASSWORD
valueFrom
:
secretKeyRef
:
name
:
"
{{
tpl
(.Values.redis.secret.name)
.
}}"
key
:
redis-password
{{
- if .Values.postgresql.enabled
}}
-
name
:
CVAT_POSTGRES_HOST
value
:
"
{{
.Release.Name
}}-postgresql"
-
name
:
CVAT_POSTGRES_PORT
value
:
"
{{
.Values.postgresql.service.ports.postgresql
}}"
{{
- else
}}
-
name
:
CVAT_POSTGRES_HOST
value
:
"
{{
.Values.postgresql.external.host
}}"
-
name
:
CVAT_POSTGRES_PORT
value
:
"
{{
.Values.postgresql.external.port
}}"
{{
- end
}}
-
name
:
CVAT_POSTGRES_USER
valueFrom
:
secretKeyRef
:
name
:
"
{{
tpl
(.Values.postgresql.secret.name)
.
}}"
key
:
username
-
name
:
CVAT_POSTGRES_DBNAME
valueFrom
:
secretKeyRef
:
name
:
"
{{
tpl
(.Values.postgresql.secret.name)
.
}}"
key
:
database
-
name
:
CVAT_POSTGRES_PASSWORD
valueFrom
:
secretKeyRef
:
name
:
"
{{
tpl
(.Values.postgresql.secret.name)
.
}}"
key
:
password
{{
- if .Values.nuclio
}}
-
name
:
CVAT_SERVERLESS
value
:
"
1"
-
name
:
CVAT_NUCLIO_HOST
value
:
"
{{
.Release.Name
}}-nuclio-dashboard"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.additionalEnv
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
volumeMounts
:
-
mountPath
:
/home/django/data
name
:
cvat-backend-data
subPath
:
data
-
mountPath
:
/home/django/keys
name
:
cvat-backend-data
subPath
:
keys
-
mountPath
:
/home/django/logs
name
:
cvat-backend-data
subPath
:
logs
-
mountPath
:
/home/django/models
name
:
cvat-backend-data
subPath
:
models
-
mountPath
:
/home/django/tmp_storage
name
:
cvat-backend-data
subPath
:
tmp_storage
{{
- with .Values.cvat.backend.worker.annotation.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
initContainers
:
{{
- if .Values.cvat.backend.permissionFix.enabled
}}
-
name
:
user-data-permission-fix
image
:
busybox
command
:
[
"
/bin/chmod"
,
"
-R"
,
"
777"
,
"
/home/django"
]
{{
- with .Values.cvat.backend.worker.annotation.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
volumeMounts
:
{{
- if .Values.cvat.backend.defaultStorage.enabled
}}
-
mountPath
:
/home/django/data
name
:
cvat-backend-data
subPath
:
data
-
mountPath
:
/home/django/keys
name
:
cvat-backend-data
subPath
:
keys
-
mountPath
:
/home/django/logs
name
:
cvat-backend-data
subPath
:
logs
-
mountPath
:
/home/django/models
name
:
cvat-backend-data
subPath
:
models
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.affinity
}}
affinity
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.tolerations
}}
tolerations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
volumes
:
{{
- if .Values.cvat.backend.defaultStorage.enabled
}}
-
name
:
cvat-backend-data
persistentVolumeClaim
:
claimName
:
"
{{
.Release.Name
}}-backend-data"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.annotation.additionalVolumes
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.imagePullSecrets
}}
imagePullSecrets
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
helm-chart/templates/cvat_backend/worker_
low
/deployment.yml
→
helm-chart/templates/cvat_backend/worker_
export
/deployment.yml
浏览文件 @
fd7d8024
apiVersion
:
apps/v1
kind
:
Deployment
metadata
:
name
:
{{
.Release.Name
}}
-backend-worker-
low
name
:
{{
.Release.Name
}}
-backend-worker-
export
namespace
:
{{
.Release.Namespace
}}
labels
:
app
:
cvat-app
tier
:
backend
component
:
worker-
low
component
:
worker-
export
{{
- include "cvat.labels" . | nindent 4
}}
{{
- with .Values.cvat.backend.worker.
low
.labels
}}
{{
- with .Values.cvat.backend.worker.
export
.labels
}}
{{
- toYaml . | nindent 4
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.annotations
}}
{{
- with .Values.cvat.backend.worker.
export
.annotations
}}
annotations
:
{{
- toYaml . | nindent 4
}}
{{
- end
}}
spec
:
replicas
:
{{
.Values.cvat.backend.worker.
low
.replicas
}}
replicas
:
{{
.Values.cvat.backend.worker.
export
.replicas
}}
strategy
:
type
:
Recreate
selector
:
matchLabels
:
{{
- include "cvat.labels" . | nindent 6
}}
{{
- with .Values.cvat.backend.worker.
low
.labels
}}
{{
- with .Values.cvat.backend.worker.
export
.labels
}}
{{
- toYaml . | nindent 6
}}
{{
- end
}}
app
:
cvat-app
tier
:
backend
component
:
worker-
low
component
:
worker-
export
template
:
metadata
:
labels
:
app
:
cvat-app
tier
:
backend
component
:
worker-
low
component
:
worker-
export
{{
- include "cvat.labels" . | nindent 8
}}
{{
- with .Values.cvat.backend.worker.
low
.labels
}}
{{
- with .Values.cvat.backend.worker.
export
.labels
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.annotations
}}
{{
- with .Values.cvat.backend.worker.
export
.annotations
}}
annotations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
spec
:
containers
:
-
name
:
cvat-app-backend-worker-
low
-container
-
name
:
cvat-app-backend-worker-
export
-container
image
:
{{
.Values.cvat.backend.image
}}
:{{ .Values.cvat.backend.tag }}
imagePullPolicy
:
{{
.Values.cvat.backend.imagePullPolicy
}}
{{
- with .Values.cvat.backend.worker.
low
.resources
}}
{{
- with .Values.cvat.backend.worker.
export
.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
args
:
[
"
-c"
,
"
supervisord/worker.
low
.conf"
]
args
:
[
"
-c"
,
"
supervisord/worker.
export
.conf"
]
env
:
{{
- if .Values.redis.enabled
}}
-
name
:
CVAT_REDIS_HOST
...
...
@@ -65,7 +65,7 @@ spec:
secretKeyRef
:
name
:
"
{{
tpl
(.Values.redis.secret.name)
.
}}"
key
:
redis-password
{{
- if .Values.postgresql.enabled
}}
{{
- if .Values.postgresql.enabled
}}
-
name
:
CVAT_POSTGRES_HOST
value
:
"
{{
.Release.Name
}}-postgresql"
-
name
:
CVAT_POSTGRES_PORT
...
...
@@ -97,11 +97,9 @@ spec:
-
name
:
CVAT_NUCLIO_HOST
value
:
"
{{
.Release.Name
}}-nuclio-dashboard"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.additionalEnv
}}
{{
- with .Values.cvat.backend.worker.
export
.additionalEnv
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
ports
:
-
containerPort
:
8080
volumeMounts
:
-
mountPath
:
/home/django/data
name
:
cvat-backend-data
...
...
@@ -118,7 +116,7 @@ spec:
-
mountPath
:
/home/django/tmp_storage
name
:
cvat-backend-data
subPath
:
tmp_storage
{{
- with .Values.cvat.backend.worker.
low
.additionalVolumeMounts
}}
{{
- with .Values.cvat.backend.worker.
export
.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
initContainers
:
...
...
@@ -126,7 +124,7 @@ spec:
-
name
:
user-data-permission-fix
image
:
busybox
command
:
[
"
/bin/chmod"
,
"
-R"
,
"
777"
,
"
/home/django"
]
{{
- with .Values.cvat.backend.worker.
low
.resources
}}
{{
- with .Values.cvat.backend.worker.
export
.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
...
...
@@ -145,15 +143,15 @@ spec:
name
:
cvat-backend-data
subPath
:
models
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.additionalVolumeMounts
}}
{{
- with .Values.cvat.backend.worker.
export
.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.affinity
}}
{{
- with .Values.cvat.backend.worker.
export
.affinity
}}
affinity
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.tolerations
}}
{{
- with .Values.cvat.backend.worker.
export
.tolerations
}}
tolerations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
...
...
@@ -163,7 +161,7 @@ spec:
persistentVolumeClaim
:
claimName
:
"
{{
.Release.Name
}}-backend-data"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
low
.additionalVolumes
}}
{{
- with .Values.cvat.backend.worker.
export
.additionalVolumes
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.imagePullSecrets
}}
...
...
helm-chart/templates/cvat_backend/worker_
defaul
t/deployment.yml
→
helm-chart/templates/cvat_backend/worker_
impor
t/deployment.yml
浏览文件 @
fd7d8024
apiVersion
:
apps/v1
kind
:
Deployment
metadata
:
name
:
{{
.Release.Name
}}
-backend-worker-
defaul
t
name
:
{{
.Release.Name
}}
-backend-worker-
impor
t
namespace
:
{{
.Release.Namespace
}}
labels
:
app
:
cvat-app
tier
:
backend
component
:
worker-
defaul
t
component
:
worker-
impor
t
{{
- include "cvat.labels" . | nindent 4
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.labels
}}
{{
- with .Values.cvat.backend.worker.
impor
t.labels
}}
{{
- toYaml . | nindent 4
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.annotations
}}
{{
- with .Values.cvat.backend.worker.
impor
t.annotations
}}
annotations
:
{{
- toYaml . | nindent 4
}}
{{
- end
}}
spec
:
replicas
:
{{
.Values.cvat.backend.worker.
defaul
t.replicas
}}
replicas
:
{{
.Values.cvat.backend.worker.
impor
t.replicas
}}
strategy
:
type
:
Recreate
selector
:
matchLabels
:
{{
- include "cvat.labels" . | nindent 6
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.labels
}}
{{
- with .Values.cvat.backend.worker.
impor
t.labels
}}
{{
- toYaml . | nindent 6
}}
{{
- end
}}
app
:
cvat-app
-worker-default
app
:
cvat-app
tier
:
backend
component
:
worker-
defaul
t
component
:
worker-
impor
t
template
:
metadata
:
labels
:
app
:
cvat-app
-worker-default
app
:
cvat-app
tier
:
backend
component
:
worker-
defaul
t
component
:
worker-
impor
t
{{
- include "cvat.labels" . | nindent 8
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.labels
}}
{{
- with .Values.cvat.backend.worker.
impor
t.labels
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.annotations
}}
{{
- with .Values.cvat.backend.worker.
impor
t.annotations
}}
annotations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
spec
:
containers
:
-
name
:
cvat-app-backend-worker-
defaul
t-container
-
name
:
cvat-app-backend-worker-
impor
t-container
image
:
{{
.Values.cvat.backend.image
}}
:{{ .Values.cvat.backend.tag }}
imagePullPolicy
:
{{
.Values.cvat.backend.imagePullPolicy
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.resources
}}
{{
- with .Values.cvat.backend.worker.
impor
t.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
args
:
[
"
-c"
,
"
supervisord/worker.
defaul
t.conf"
]
args
:
[
"
-c"
,
"
supervisord/worker.
impor
t.conf"
]
env
:
{{
- if .Values.redis.enabled
}}
-
name
:
CVAT_REDIS_HOST
...
...
@@ -97,11 +97,9 @@ spec:
-
name
:
CVAT_NUCLIO_HOST
value
:
"
{{
.Release.Name
}}-nuclio-dashboard"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.additionalEnv
}}
{{
- with .Values.cvat.backend.worker.
impor
t.additionalEnv
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
ports
:
-
containerPort
:
8080
volumeMounts
:
-
mountPath
:
/home/django/data
name
:
cvat-backend-data
...
...
@@ -118,7 +116,7 @@ spec:
-
mountPath
:
/home/django/tmp_storage
name
:
cvat-backend-data
subPath
:
tmp_storage
{{
- with .Values.cvat.backend.worker.
defaul
t.additionalVolumeMounts
}}
{{
- with .Values.cvat.backend.worker.
impor
t.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
initContainers
:
...
...
@@ -126,7 +124,7 @@ spec:
-
name
:
user-data-permission-fix
image
:
busybox
command
:
[
"
/bin/chmod"
,
"
-R"
,
"
777"
,
"
/home/django"
]
{{
- with .Values.cvat.backend.worker.
defaul
t.resources
}}
{{
- with .Values.cvat.backend.worker.
impor
t.resources
}}
resources
:
{{
- toYaml . | nindent 12
}}
{{
- end
}}
...
...
@@ -145,15 +143,15 @@ spec:
name
:
cvat-backend-data
subPath
:
models
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.additionalVolumeMounts
}}
{{
- with .Values.cvat.backend.worker.
impor
t.additionalVolumeMounts
}}
{{
- toYaml . | nindent 10
}}
{{
- end
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.affinity
}}
{{
- with .Values.cvat.backend.worker.
impor
t.affinity
}}
affinity
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.tolerations
}}
{{
- with .Values.cvat.backend.worker.
impor
t.tolerations
}}
tolerations
:
{{
- toYaml . | nindent 8
}}
{{
- end
}}
...
...
@@ -163,7 +161,7 @@ spec:
persistentVolumeClaim
:
claimName
:
"
{{
.Release.Name
}}-backend-data"
{{
- end
}}
{{
- with .Values.cvat.backend.worker.
defaul
t.additionalVolumes
}}
{{
- with .Values.cvat.backend.worker.
impor
t.additionalVolumes
}}
{{
- toYaml . | nindent 8
}}
{{
- end
}}
{{
- with .Values.imagePullSecrets
}}
...
...
helm-chart/values.yaml
浏览文件 @
fd7d8024
...
...
@@ -32,7 +32,7 @@ cvat:
additionalVolumes
:
[]
additionalVolumeMounts
:
[]
worker
:
defaul
t
:
expor
t
:
replicas
:
2
labels
:
{}
annotations
:
{}
...
...
@@ -42,7 +42,17 @@ cvat:
additionalEnv
:
[]
additionalVolumes
:
[]
additionalVolumeMounts
:
[]
low
:
import
:
replicas
:
2
labels
:
{}
annotations
:
{}
resources
:
{}
affinity
:
{}
tolerations
:
[]
additionalEnv
:
[]
additionalVolumes
:
[]
additionalVolumeMounts
:
[]
annotation
:
replicas
:
1
labels
:
{}
annotations
:
{}
...
...
supervisord/all.conf
浏览文件 @
fd7d8024
...
...
@@ -22,16 +22,22 @@ command=bash -c "rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-
priority
=
1
autorestart
=
true
[
program
:
rqworker_
defaul
t
]
[
program
:
rqworker_
expor
t
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
\
"exec python3 %(ENV_HOME)s/manage.py rqworker -v 3
defaul
t"
"exec python3 %(ENV_HOME)s/manage.py rqworker -v 3
expor
t"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=
2
process_name
=
rqworker_default_
%(
process_num
)
s
[
program
:
rqworker_low
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
\
"exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 low"
"exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 import"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=
2
process_name
=
rqworker_default_
%(
process_num
)
s
[
program
:
rqworker_annotation
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
\
"exec python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=
1
...
...
@@ -49,7 +55,7 @@ numprocs=1
[
program
:
rqscheduler
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
\
"python3 /opt/venv/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -i 30"
"python3 /opt/venv/bin/rqscheduler --host %(ENV_CVAT_REDIS_HOST)s -
-password '%(ENV_CVAT_REDIS_PASSWORD)s' -
i 30"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=
1
...
...
supervisord/worker.
default
.conf
→
supervisord/worker.
annotation
.conf
浏览文件 @
fd7d8024
...
...
@@ -22,18 +22,10 @@ command=bash -c "rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-
priority
=
1
autorestart
=
true
[
program
:
rqworker_
default
]
[
program
:
rqworker_
annotation
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
"
\
exec
python3
%(
ENV_HOME
)
s
/
manage
.
py
rqworker
-
v
3
default
\
exec
python3
%(
ENV_HOME
)
s
/
manage
.
py
rqworker
-
v
3
annotation
\
--
worker
-
class
cvat
.
rqworker
.
DefaultWorker
\
"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=%(
ENV_NUMPROCS
)
s
process_name
=
rqworker_default_
%(
process_num
)
s
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
[
program
:
clamav_update
]
command
=
bash
-
c
"if [ \"
${
CLAM_AV
}\
"
=
'yes'
];
then
/
usr
/
bin
/
freshclam
-
d
\
-
l
%(
ENV_HOME
)
s
/
logs
/
freshclam
.
log
--
foreground
=
true
;
fi
"
numprocs
=
1
supervisord/worker.export.conf
0 → 100644
浏览文件 @
fd7d8024
[
unix_http_server
]
file
= /
tmp
/
supervisord
/
supervisor
.
sock
[
supervisorctl
]
serverurl
=
unix
:///
tmp
/
supervisord
/
supervisor
.
sock
[
rpcinterface
:
supervisor
]
supervisor
.
rpcinterface_factory
=
supervisor
.
rpcinterface
:
make_main_rpcinterface
[
supervisord
]
nodaemon
=
true
logfile
=%(
ENV_HOME
)
s
/
logs
/
supervisord
.
log
;
supervisord
log
file
logfile_maxbytes
=
50
MB
;
maximum
size
of
logfile
before
rotation
logfile_backups
=
10
;
number
of
backed
up
logfiles
loglevel
=
debug
;
info
,
debug
,
warn
,
trace
pidfile
=/
tmp
/
supervisord
/
supervisord
.
pid
;
pidfile
location
childlogdir
=%(
ENV_HOME
)
s
/
logs
/ ;
where
child
log
files
will
live
[
program
:
ssh
-
agent
]
command
=
bash
-
c
"rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-agent.sock"
priority
=
1
autorestart
=
true
[
program
:
rqworker_export
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
"
\
exec
python3
%(
ENV_HOME
)
s
/
manage
.
py
rqworker
-
v
3
export
\
--
worker
-
class
cvat
.
rqworker
.
DefaultWorker
\
"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=%(
ENV_NUMPROCS
)
s
process_name
=
rqworker_export_
%(
process_num
)
s
supervisord/worker.
low
.conf
→
supervisord/worker.
import
.conf
浏览文件 @
fd7d8024
...
...
@@ -22,17 +22,18 @@ command=bash -c "rm /tmp/ssh-agent.sock -f && /usr/bin/ssh-agent -d -a /tmp/ssh-
priority
=
1
autorestart
=
true
[
program
:
rqworker_
low
]
[
program
:
rqworker_
import
]
command
=%(
ENV_HOME
)
s
/
wait
-
for
-
it
.
sh
%(
ENV_CVAT_REDIS_HOST
)
s
:
6379
-
t
0
--
bash
-
ic
"
\
exec
python3
%(
ENV_HOME
)
s
/
manage
.
py
rqworker
-
v
3
low
\
exec
python3
%(
ENV_HOME
)
s
/
manage
.
py
rqworker
-
v
3
import
\
--
worker
-
class
cvat
.
rqworker
.
DefaultWorker
\
"
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
numprocs
=%(
ENV_NUMPROCS
)
s
process_name
=
rqworker_import_
%(
process_num
)
s
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
[
program
:
clamav_update
]
command
=
bash
-
c
"if [ \"
${
CLAM_AV
}\
"
=
'yes'
];
then
/
usr
/
bin
/
freshclam
-
d
\
-
l
%(
ENV_HOME
)
s
/
logs
/
freshclam
.
log
--
foreground
=
true
;
fi
"
numprocs
=
1
environment
=
SSH_AUTH_SOCK
=
"/tmp/ssh-agent.sock"
tests/docker-compose.file_share.yml
浏览文件 @
fd7d8024
services
:
cvat_worker_
defaul
t
:
cvat_worker_
impor
t
:
volumes
:
-
./tests/mounted_file_share:/home/django/share:rw
cvat_server
:
...
...
tests/values.test.yaml
浏览文件 @
fd7d8024
...
...
@@ -6,7 +6,7 @@ cvat:
name
:
cvat-backend-data
subPath
:
share
worker
:
defaul
t
:
impor
t
:
additionalVolumeMounts
:
-
mountPath
:
/home/django/share
name
:
cvat-backend-data
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录