Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
疯人忠
Cvat
提交
bceae228
C
Cvat
项目概览
疯人忠
/
Cvat
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
Cvat
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
bceae228
编写于
3月 22, 2022
作者:
K
Kirill Lakhov
提交者:
GitHub
3月 22, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Tus for project dataset (#4485)
上级
dd60b2d8
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
115 addition
and
65 deletion
+115
-65
CHANGELOG.md
CHANGELOG.md
+1
-0
cvat-core/package-lock.json
cvat-core/package-lock.json
+2
-2
cvat-core/package.json
cvat-core/package.json
+1
-1
cvat-core/src/server-proxy.js
cvat-core/src/server-proxy.js
+52
-31
cvat/apps/engine/models.py
cvat/apps/engine/models.py
+3
-0
cvat/apps/engine/views.py
cvat/apps/engine/views.py
+54
-31
cvat/apps/iam/permissions.py
cvat/apps/iam/permissions.py
+2
-0
未找到文件。
CHANGELOG.md
浏览文件 @
bceae228
...
...
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
-
Task annotations importing via chunk uploads (
<https://github.com/openvinotoolkit/cvat/pull/4327>
)
-
Advanced filtration and sorting for a list of tasks/projects/cloudstorages (
<https://github.com/openvinotoolkit/cvat/pull/4403>
)
-
Project dataset importing via chunk uploads (
<https://github.com/openvinotoolkit/cvat/pull/4485>
)
### Changed
-
Added missing geos dependency into Dockerfile (
<https://github.com/openvinotoolkit/cvat/pull/4451>
)
...
...
cvat-core/package-lock.json
浏览文件 @
bceae228
{
"name"
:
"cvat-core"
,
"version"
:
"5.0.
0
"
,
"version"
:
"5.0.
1
"
,
"lockfileVersion"
:
2
,
"requires"
:
true
,
"packages"
:
{
""
:
{
"name"
:
"cvat-core"
,
"version"
:
"5.0.
0
"
,
"version"
:
"5.0.
1
"
,
"license"
:
"MIT"
,
"dependencies"
:
{
"axios"
:
"^0.21.4"
,
...
...
cvat-core/package.json
浏览文件 @
bceae228
{
"name"
:
"cvat-core"
,
"version"
:
"5.0.
0
"
,
"version"
:
"5.0.
1
"
,
"description"
:
"Part of Computer Vision Tool which presents an interface for client-side integration"
,
"main"
:
"babel.config.js"
,
"scripts"
:
{
...
...
cvat-core/src/server-proxy.js
浏览文件 @
bceae228
...
...
@@ -75,7 +75,7 @@
onProgress
(
bytesUploaded
)
{
if
(
onUpdate
&&
Number
.
isInteger
(
totalSentSize
)
&&
Number
.
isInteger
(
totalSize
))
{
const
currentUploadedSize
=
totalSentSize
+
bytesUploaded
;
const
percentage
=
currentUploadedSize
/
totalSize
;
const
percentage
=
Math
.
round
(
currentUploadedSize
/
totalSize
)
;
onUpdate
(
percentage
);
}
},
...
...
@@ -612,41 +612,63 @@
}
async
function
importDataset
(
id
,
format
,
file
,
onUpdate
)
{
const
{
backendAPI
}
=
config
;
const
{
backendAPI
,
origin
}
=
config
;
const
params
=
{
...
enableOrganization
(),
format
,
filename
:
file
.
name
,
};
const
uploadConfig
=
{
chunkSize
:
config
.
uploadChunkSize
*
1024
*
1024
,
endpoint
:
`
${
origin
}${
backendAPI
}
/projects/
${
id
}
/dataset/`
,
totalSentSize
:
0
,
totalSize
:
file
.
size
,
onUpdate
:
(
percentage
)
=>
{
onUpdate
(
'
The dataset is being uploaded to the server
'
,
percentage
);
},
};
const
url
=
`
${
backendAPI
}
/projects/
${
id
}
/dataset`
;
const
formData
=
new
FormData
();
formData
.
append
(
'
dataset_file
'
,
file
);
return
new
Promise
((
resolve
,
reject
)
=>
{
async
function
requestStatus
()
{
try
{
const
response
=
await
Axios
.
get
(
`
${
url
}
?action=import_status`
,
{
proxy
:
config
.
proxy
,
});
if
(
response
.
status
===
202
)
{
if
(
onUpdate
&&
response
.
data
.
message
!==
''
)
{
onUpdate
(
response
.
data
.
message
,
response
.
data
.
progress
||
0
);
try
{
await
Axios
.
post
(
url
,
new
FormData
(),
{
params
,
proxy
:
config
.
proxy
,
headers
:
{
'
Upload-Start
'
:
true
},
});
await
chunkUpload
(
file
,
uploadConfig
);
await
Axios
.
post
(
url
,
new
FormData
(),
{
params
,
proxy
:
config
.
proxy
,
headers
:
{
'
Upload-Finish
'
:
true
},
});
return
new
Promise
((
resolve
,
reject
)
=>
{
async
function
requestStatus
()
{
try
{
const
response
=
await
Axios
.
get
(
url
,
{
params
:
{
...
params
,
action
:
'
import_status
'
},
proxy
:
config
.
proxy
,
});
if
(
response
.
status
===
202
)
{
if
(
onUpdate
&&
response
.
data
.
message
)
{
onUpdate
(
response
.
data
.
message
,
response
.
data
.
progress
||
0
);
}
setTimeout
(
requestStatus
,
3000
);
}
else
if
(
response
.
status
===
201
)
{
resolve
();
}
else
{
reject
(
generateError
(
response
));
}
setTimeout
(
requestStatus
,
3000
);
}
else
if
(
response
.
status
===
201
)
{
resolve
();
}
else
{
reject
(
generateError
(
response
));
}
catch
(
error
)
{
reject
(
generateError
(
error
));
}
}
catch
(
error
)
{
reject
(
generateError
(
error
));
}
}
Axios
.
post
(
`
${
url
}
?format=
${
format
}
`
,
formData
,
{
proxy
:
config
.
proxy
,
}).
then
(()
=>
{
setTimeout
(
requestStatus
,
2000
);
}).
catch
((
error
)
=>
{
reject
(
generateError
(
error
));
});
});
}
catch
(
errorData
)
{
throw
generateError
(
errorData
);
}
}
async
function
exportTask
(
id
)
{
...
...
@@ -1279,8 +1301,7 @@
setTimeout
(
requestStatus
);
});
}
catch
(
errorData
)
{
generateError
(
errorData
);
return
null
;
throw
generateError
(
errorData
);
}
}
...
...
cvat/apps/engine/models.py
浏览文件 @
bceae228
...
...
@@ -248,6 +248,9 @@ class Project(models.Model):
def
get_project_logs_dirname
(
self
):
return
os
.
path
.
join
(
self
.
get_project_dirname
(),
'logs'
)
def
get_tmp_dirname
(
self
):
return
os
.
path
.
join
(
self
.
get_project_dirname
(),
"tmp"
)
def
get_client_log_path
(
self
):
return
os
.
path
.
join
(
self
.
get_project_logs_dirname
(),
"client.log"
)
...
...
cvat/apps/engine/views.py
浏览文件 @
bceae228
...
...
@@ -250,7 +250,7 @@ class ServerViewSet(viewsets.ViewSet):
'200'
:
ProjectSerializer
,
})
)
class
ProjectViewSet
(
viewsets
.
ModelViewSet
):
class
ProjectViewSet
(
viewsets
.
ModelViewSet
,
UploadMixin
):
queryset
=
models
.
Project
.
objects
.
prefetch_related
(
Prefetch
(
'label_set'
,
queryset
=
models
.
Label
.
objects
.
order_by
(
'id'
)
))
...
...
@@ -330,21 +330,13 @@ class ProjectViewSet(viewsets.ModelViewSet):
'400'
:
OpenApiResponse
(
description
=
'Failed to import dataset'
),
'405'
:
OpenApiResponse
(
description
=
'Format is not available'
),
})
@
action
(
detail
=
True
,
methods
=
[
'GET'
,
'POST'
],
serializer_class
=
None
,
url_path
=
'dataset
'
)
@
action
(
detail
=
True
,
methods
=
[
'GET'
,
'POST'
,
'OPTIONS'
],
serializer_class
=
None
,
url_path
=
r
'dataset/?$
'
)
def
dataset
(
self
,
request
,
pk
):
db_project
=
self
.
get_object
()
# force to call check_object_permissions
if
request
.
method
==
'POST'
:
format_name
=
request
.
query_params
.
get
(
"format"
,
""
)
self
.
_object
=
self
.
get_object
()
# force to call check_object_permissions
return
_import_project_dataset
(
request
=
request
,
rq_id
=
f
"/api/project/
{
pk
}
/dataset_import"
,
rq_func
=
dm
.
project
.
import_dataset_as_project
,
pk
=
pk
,
format_name
=
format_name
,
)
if
request
.
method
==
'POST'
or
request
.
method
==
'OPTIONS'
:
return
self
.
upload_data
(
request
)
else
:
action
=
request
.
query_params
.
get
(
"action"
,
""
).
lower
()
if
action
in
(
"import_status"
,):
...
...
@@ -353,12 +345,12 @@ class ProjectViewSet(viewsets.ModelViewSet):
if
rq_job
is
None
:
return
Response
(
status
=
status
.
HTTP_404_NOT_FOUND
)
elif
rq_job
.
is_finished
:
os
.
close
(
rq_job
.
meta
[
'tmp_file_descriptor'
])
if
rq_job
.
meta
[
'tmp_file_descriptor'
]:
os
.
close
(
rq_job
.
meta
[
'tmp_file_descriptor'
])
os
.
remove
(
rq_job
.
meta
[
'tmp_file'
])
rq_job
.
delete
()
return
Response
(
status
=
status
.
HTTP_201_CREATED
)
elif
rq_job
.
is_failed
:
os
.
close
(
rq_job
.
meta
[
'tmp_file_descriptor'
])
if
rq_job
.
meta
[
'tmp_file_descriptor'
]:
os
.
close
(
rq_job
.
meta
[
'tmp_file_descriptor'
])
os
.
remove
(
rq_job
.
meta
[
'tmp_file'
])
rq_job
.
delete
()
return
Response
(
...
...
@@ -373,7 +365,7 @@ class ProjectViewSet(viewsets.ModelViewSet):
else
:
format_name
=
request
.
query_params
.
get
(
"format"
,
""
)
return
_export_annotations
(
db_instance
=
db_pro
ject
,
db_instance
=
self
.
_ob
ject
,
rq_id
=
"/api/project/{}/dataset/{}"
.
format
(
pk
,
format_name
),
request
=
request
,
action
=
action
,
...
...
@@ -382,6 +374,35 @@ class ProjectViewSet(viewsets.ModelViewSet):
filename
=
request
.
query_params
.
get
(
"filename"
,
""
).
lower
(),
)
def
get_upload_dir
(
self
):
if
'dataset'
in
self
.
action
:
return
self
.
_object
.
get_tmp_dirname
()
return
""
def
upload_finished
(
self
,
request
):
if
self
.
action
==
'dataset'
:
format_name
=
request
.
query_params
.
get
(
"format"
,
""
)
filename
=
request
.
query_params
.
get
(
"filename"
,
""
)
tmp_dir
=
self
.
_object
.
get_tmp_dirname
()
uploaded_file
=
None
if
os
.
path
.
isfile
(
os
.
path
.
join
(
tmp_dir
,
filename
)):
uploaded_file
=
os
.
path
.
join
(
tmp_dir
,
filename
)
return
_import_project_dataset
(
request
=
request
,
filename
=
uploaded_file
,
rq_id
=
f
"/api/project/
{
self
.
_object
.
pk
}
/dataset_import"
,
rq_func
=
dm
.
project
.
import_dataset_as_project
,
pk
=
self
.
_object
.
pk
,
format_name
=
format_name
,
)
return
Response
(
data
=
'Unknown upload was finished'
,
status
=
status
.
HTTP_400_BAD_REQUEST
)
@
action
(
detail
=
True
,
methods
=
[
'HEAD'
,
'PATCH'
],
url_path
=
'dataset/'
+
UploadMixin
.
file_id_regex
)
def
append_dataset_chunk
(
self
,
request
,
pk
,
file_id
):
self
.
_object
=
self
.
get_object
()
return
self
.
append_tus_chunk
(
request
,
file_id
)
@
extend_schema
(
summary
=
'Method allows to download project annotations'
,
parameters
=
[
OpenApiParameter
(
'format'
,
description
=
'Desired output format name
\n
'
...
...
@@ -1768,7 +1789,7 @@ def _export_annotations(db_instance, rq_id, request, format_name, action, callba
result_ttl
=
ttl
,
failure_ttl
=
ttl
)
return
Response
(
status
=
status
.
HTTP_202_ACCEPTED
)
def
_import_project_dataset
(
request
,
rq_id
,
rq_func
,
pk
,
format_name
):
def
_import_project_dataset
(
request
,
rq_id
,
rq_func
,
pk
,
format_name
,
filename
=
None
):
format_desc
=
{
f
.
DISPLAY_NAME
:
f
for
f
in
dm
.
views
.
get_import_formats
()}.
get
(
format_name
)
if
format_desc
is
None
:
...
...
@@ -1781,19 +1802,21 @@ def _import_project_dataset(request, rq_id, rq_func, pk, format_name):
rq_job
=
queue
.
fetch_job
(
rq_id
)
if
not
rq_job
:
serializer
=
DatasetFileSerializer
(
data
=
request
.
data
)
if
serializer
.
is_valid
(
raise_exception
=
True
):
dataset_file
=
serializer
.
validated_data
[
'dataset_file'
]
fd
,
filename
=
mkstemp
(
prefix
=
'cvat_{}'
.
format
(
pk
))
with
open
(
filename
,
'wb+'
)
as
f
:
for
chunk
in
dataset_file
.
chunks
():
f
.
write
(
chunk
)
rq_job
=
queue
.
enqueue_call
(
func
=
rq_func
,
args
=
(
pk
,
filename
,
format_name
),
job_id
=
rq_id
,
meta
=
{
fd
=
None
if
not
filename
:
serializer
=
DatasetFileSerializer
(
data
=
request
.
data
)
if
serializer
.
is_valid
(
raise_exception
=
True
):
dataset_file
=
serializer
.
validated_data
[
'dataset_file'
]
fd
,
filename
=
mkstemp
(
prefix
=
'cvat_{}'
.
format
(
pk
))
with
open
(
filename
,
'wb+'
)
as
f
:
for
chunk
in
dataset_file
.
chunks
():
f
.
write
(
chunk
)
rq_job
=
queue
.
enqueue_call
(
func
=
rq_func
,
args
=
(
pk
,
filename
,
format_name
),
job_id
=
rq_id
,
meta
=
{
'tmp_file'
:
filename
,
'tmp_file_descriptor'
:
fd
,
},
...
...
cvat/apps/iam/permissions.py
浏览文件 @
bceae228
...
...
@@ -492,6 +492,8 @@ class ProjectPermission(OpenPolicyAgentPermission):
(
'retrieve'
,
'GET'
):
'view'
,
(
'tasks'
,
'GET'
):
'view'
,
(
'dataset'
,
'POST'
):
'import:dataset'
,
(
'append_dataset_chunk'
,
'HEAD'
):
'import:dataset'
,
(
'append_dataset_chunk'
,
'PATCH'
):
'import:dataset'
,
(
'annotations'
,
'GET'
):
'export:annotations'
,
(
'dataset'
,
'GET'
):
'export:dataset'
,
(
'export_backup'
,
'GET'
):
'export:backup'
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录