Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
疯人忠
Cvat
提交
182c39aa
C
Cvat
项目概览
疯人忠
/
Cvat
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
Cvat
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
182c39aa
编写于
8月 18, 2022
作者:
M
Maria Khrustaleva
提交者:
GitHub
8月 18, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Hotfix missing manifest file (#116)
上级
5e2eda76
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
32 addition
and
26 deletion
+32
-26
CHANGELOG.md
CHANGELOG.md
+1
-0
cvat/apps/engine/task.py
cvat/apps/engine/task.py
+23
-21
tests/python/rest_api/test_tasks.py
tests/python/rest_api/test_tasks.py
+8
-5
未找到文件。
CHANGELOG.md
浏览文件 @
182c39aa
...
...
@@ -50,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
-
Annotation window might have top offset if try to move a locked object
-
Image search in cloud storage (
<https://github.com/cvat-ai/cvat/pull/8>
)
-
Reset password functionality (
<https://github.com/cvat-ai/cvat/pull/52>
)
-
Creating task with cloud storage data (
<https://github.com/cvat-ai/cvat/pull/116>
)
### Security
-
TDB
...
...
cvat/apps/engine/task.py
浏览文件 @
182c39aa
# Copyright (C) 2018-2022 Intel Corporation
# Copyright (C) 2022 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT
...
...
@@ -18,9 +19,11 @@ import requests
import
ipaddress
import
dns.resolver
import
django_rq
import
pytz
from
django.conf
import
settings
from
django.db
import
transaction
from
datetime
import
datetime
from
cvat.apps.engine
import
models
from
cvat.apps.engine.log
import
slogger
...
...
@@ -30,7 +33,7 @@ from cvat.apps.engine.utils import av_scan_paths
from
utils.dataset_manifest
import
ImageManifestManager
,
VideoManifestManager
,
is_manifest
from
utils.dataset_manifest.core
import
VideoManifestValidator
from
utils.dataset_manifest.utils
import
detect_related_images
from
.cloud_provider
import
get_cloud_storage_instance
,
Credentials
from
.cloud_provider
import
db_storage_to_storage_instance
############################# Low Level server API
...
...
@@ -207,13 +210,21 @@ def _validate_data(counter, manifest_files=None):
return
counter
,
task_modes
[
0
]
def
_validate_manifest
(
manifests
,
root_dir
):
def
_validate_manifest
(
manifests
,
root_dir
,
is_in_cloud
,
db_cloud_storage
):
if
manifests
:
if
len
(
manifests
)
!=
1
:
raise
Exception
(
'Only one manifest file can be attached with data'
)
manifest_file
=
manifests
[
0
]
full_manifest_path
=
os
.
path
.
join
(
root_dir
,
manifests
[
0
])
if
is_in_cloud
:
cloud_storage_instance
=
db_storage_to_storage_instance
(
db_cloud_storage
)
# check that cloud storage manifest file exists and is up to date
if
not
os
.
path
.
exists
(
full_manifest_path
)
or
\
datetime
.
utcfromtimestamp
(
os
.
path
.
getmtime
(
full_manifest_path
)).
replace
(
tzinfo
=
pytz
.
UTC
)
\
<
cloud_storage_instance
.
get_file_last_modified
(
manifest_file
):
cloud_storage_instance
.
download_file
(
manifest_file
,
full_manifest_path
)
if
is_manifest
(
full_manifest_path
):
return
manifest
s
[
0
]
return
manifest
_file
raise
Exception
(
'Invalid manifest was uploaded'
)
return
None
...
...
@@ -293,6 +304,7 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False):
db_data
=
db_task
.
data
upload_dir
=
db_data
.
get_upload_dirname
()
is_data_in_cloud
=
db_data
.
storage
==
models
.
StorageChoice
.
CLOUD_STORAGE
if
data
[
'remote_files'
]
and
not
isDatasetImport
:
data
[
'remote_files'
]
=
_download_data
(
data
[
'remote_files'
],
upload_dir
)
...
...
@@ -310,28 +322,18 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False):
manifest_root
=
None
if
db_data
.
storage
in
{
models
.
StorageChoice
.
LOCAL
,
models
.
StorageChoice
.
SHARE
}:
manifest_root
=
upload_dir
elif
db_data
.
storage
==
models
.
StorageChoice
.
CLOUD_STORAGE
:
elif
is_data_in_cloud
:
manifest_root
=
db_data
.
cloud_storage
.
get_storage_dirname
()
manifest_file
=
_validate_manifest
(
manifest_files
,
manifest_root
)
manifest_file
=
_validate_manifest
(
manifest_files
,
manifest_root
,
is_data_in_cloud
,
db_data
.
cloud_storage
if
is_data_in_cloud
else
None
)
if
manifest_file
and
(
not
settings
.
USE_CACHE
or
db_data
.
storage_method
!=
models
.
StorageMethodChoice
.
CACHE
):
raise
Exception
(
"File with meta information can be uploaded if 'Use cache' option is also selected"
)
if
data
[
'server_files'
]
and
db_data
.
storage
==
models
.
StorageChoice
.
CLOUD_STORAGE
:
if
not
manifest_file
:
raise
Exception
(
'A manifest file not found'
)
db_cloud_storage
=
db_data
.
cloud_storage
credentials
=
Credentials
()
credentials
.
convert_from_db
({
'type'
:
db_cloud_storage
.
credentials_type
,
'value'
:
db_cloud_storage
.
credentials
,
})
details
=
{
'resource'
:
db_cloud_storage
.
resource
,
'credentials'
:
credentials
,
'specific_attributes'
:
db_cloud_storage
.
get_specific_attributes
()
}
cloud_storage_instance
=
get_cloud_storage_instance
(
cloud_provider
=
db_cloud_storage
.
provider_type
,
**
details
)
if
data
[
'server_files'
]
and
is_data_in_cloud
:
cloud_storage_instance
=
db_storage_to_storage_instance
(
db_data
.
cloud_storage
)
sorted_media
=
sort
(
media
[
'image'
],
data
[
'sorting_method'
])
data_size
=
len
(
sorted_media
)
...
...
@@ -516,7 +518,7 @@ def _create_thread(db_task, data, isBackupRestore=False, isDatasetImport=False):
# calculate chunk size if it isn't specified
if
db_data
.
chunk_size
is
None
:
if
isinstance
(
compressed_chunk_writer
,
ZipCompressedChunkWriter
):
if
not
db_data
.
storage
==
models
.
StorageChoice
.
CLOUD_STORAGE
:
if
not
is_data_in_cloud
:
w
,
h
=
extractor
.
get_image_size
(
0
)
else
:
img_properties
=
manifest
[
0
]
...
...
tests/python/rest_api/test_tasks.py
浏览文件 @
182c39aa
...
...
@@ -320,12 +320,15 @@ class TestPostTaskData:
(
task
,
_
)
=
api_client
.
tasks_api
.
retrieve
(
task_id
)
assert
task
.
size
==
4
@
pytest
.
mark
.
parametrize
(
'cloud_storage_id, manifest, org'
,
[
(
1
,
'manifest.jsonl'
,
''
),
# public bucket
(
2
,
'sub/manifest.jsonl'
,
'org2'
),
# private bucket
@
pytest
.
mark
.
parametrize
(
'cloud_storage_id, manifest,
use_bucket_content,
org'
,
[
(
1
,
'manifest.jsonl'
,
False
,
''
),
# public bucket
(
2
,
'sub/manifest.jsonl'
,
True
,
'org2'
),
# private bucket
])
def
test_create_task_with_cloud_storage_files
(
self
,
cloud_storage_id
,
manifest
,
org
):
cloud_storage_content
=
get_cloud_storage_content
(
self
.
_USERNAME
,
cloud_storage_id
,
manifest
)
def
test_create_task_with_cloud_storage_files
(
self
,
cloud_storage_id
,
manifest
,
use_bucket_content
,
org
):
if
use_bucket_content
:
cloud_storage_content
=
get_cloud_storage_content
(
self
.
_USERNAME
,
cloud_storage_id
,
manifest
)
else
:
cloud_storage_content
=
[
'image_case_65_1.png'
,
'image_case_65_2.png'
]
cloud_storage_content
.
append
(
manifest
)
task_spec
=
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录