Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
qq_35024513
milvus
提交
4e8ba4e0
milvus
项目概览
qq_35024513
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
14
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
4e8ba4e0
编写于
7月 08, 2020
作者:
F
fishpenguin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Server crashed when calling get_entity_by_id
Signed-off-by:
N
fishpenguin
<
kun.yu@zilliz.com
>
上级
5382d81a
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
33 addition
and
78 deletion
+33
-78
CHANGELOG.md
CHANGELOG.md
+1
-0
core/src/db/DB.h
core/src/db/DB.h
+0
-9
core/src/db/DBImpl.cpp
core/src/db/DBImpl.cpp
+4
-41
core/src/db/DBImpl.h
core/src/db/DBImpl.h
+0
-9
core/src/server/grpc_impl/GrpcRequestHandler.cpp
core/src/server/grpc_impl/GrpcRequestHandler.cpp
+2
-2
sdk/examples/simple/src/ClientTest.cpp
sdk/examples/simple/src/ClientTest.cpp
+18
-12
sdk/examples/utils/Utils.cpp
sdk/examples/utils/Utils.cpp
+6
-3
tests/milvus_python_test/entity/test_get_entity_by_id.py
tests/milvus_python_test/entity/test_get_entity_by_id.py
+2
-2
未找到文件。
CHANGELOG.md
浏览文件 @
4e8ba4e0
...
...
@@ -25,6 +25,7 @@ Please mark all changes in change log and use the issue from GitHub
-
\#
2731 No entity returned with
`get_entity_by_id`
-
\#
2732 Server destroyed after
`delete by id`
-
\#
2733 The max value of top-k should be limited
-
\#
2765 Server crashed when calling
`get_entity_by_id`
## Feature
-
\#
2319 Redo metadata to support MVCC
...
...
core/src/db/DB.h
浏览文件 @
4e8ba4e0
...
...
@@ -100,15 +100,6 @@ class DB {
virtual
Status
InsertVectors
(
const
std
::
string
&
collection_id
,
const
std
::
string
&
partition_tag
,
VectorsData
&
vectors
)
=
0
;
virtual
Status
DeleteVector
(
const
std
::
string
&
collection_id
,
IDNumber
vector_id
)
=
0
;
virtual
Status
DeleteEntity
(
const
std
::
string
&
collection_id
,
IDNumber
entity_id
)
=
0
;
virtual
Status
DeleteVectors
(
const
std
::
string
&
collection_id
,
IDNumbers
vector_ids
)
=
0
;
virtual
Status
DeleteEntities
(
const
std
::
string
&
collection_id
,
IDNumbers
entity_ids
)
=
0
;
...
...
core/src/db/DBImpl.cpp
浏览文件 @
4e8ba4e0
...
...
@@ -802,7 +802,7 @@ CopyToAttr(const std::vector<uint8_t>& record, int64_t row_num, const std::vecto
attr_nbytes
.
insert
(
std
::
make_pair
(
name
,
sizeof
(
int8_t
)));
attr_data_size
.
insert
(
std
::
make_pair
(
name
,
row_num
*
sizeof
(
int8_t
)));
offset
+=
row_num
*
sizeof
(
int
64
_t
);
offset
+=
row_num
*
sizeof
(
int
32
_t
);
break
;
}
case
meta
::
hybrid
::
DataType
::
INT16
:
{
...
...
@@ -822,7 +822,7 @@ CopyToAttr(const std::vector<uint8_t>& record, int64_t row_num, const std::vecto
attr_nbytes
.
insert
(
std
::
make_pair
(
name
,
sizeof
(
int16_t
)));
attr_data_size
.
insert
(
std
::
make_pair
(
name
,
row_num
*
sizeof
(
int16_t
)));
offset
+=
row_num
*
sizeof
(
int
64
_t
);
offset
+=
row_num
*
sizeof
(
int
32
_t
);
break
;
}
case
meta
::
hybrid
::
DataType
::
INT32
:
{
...
...
@@ -837,7 +837,7 @@ CopyToAttr(const std::vector<uint8_t>& record, int64_t row_num, const std::vecto
attr_nbytes
.
insert
(
std
::
make_pair
(
name
,
sizeof
(
int32_t
)));
attr_data_size
.
insert
(
std
::
make_pair
(
name
,
row_num
*
sizeof
(
int32_t
)));
offset
+=
row_num
*
sizeof
(
int
64
_t
);
offset
+=
row_num
*
sizeof
(
int
32
_t
);
break
;
}
case
meta
::
hybrid
::
DataType
::
INT64
:
{
...
...
@@ -866,7 +866,7 @@ CopyToAttr(const std::vector<uint8_t>& record, int64_t row_num, const std::vecto
attr_nbytes
.
insert
(
std
::
make_pair
(
name
,
sizeof
(
float
)));
attr_data_size
.
insert
(
std
::
make_pair
(
name
,
row_num
*
sizeof
(
float
)));
offset
+=
row_num
*
sizeof
(
double
);
offset
+=
row_num
*
sizeof
(
float
);
break
;
}
case
meta
::
hybrid
::
DataType
::
DOUBLE
:
{
...
...
@@ -984,43 +984,6 @@ DBImpl::InsertEntities(const std::string& collection_id, const std::string& part
return
status
;
}
Status
DBImpl
::
DeleteVector
(
const
std
::
string
&
collection_id
,
IDNumber
vector_id
)
{
IDNumbers
ids
;
ids
.
push_back
(
vector_id
);
return
DeleteVectors
(
collection_id
,
ids
);
}
Status
DBImpl
::
DeleteEntity
(
const
std
::
string
&
collection_id
,
milvus
::
engine
::
IDNumber
entity_id
)
{
IDNumbers
ids
;
ids
.
push_back
(
entity_id
);
}
Status
DBImpl
::
DeleteVectors
(
const
std
::
string
&
collection_id
,
IDNumbers
vector_ids
)
{
if
(
!
initialized_
.
load
(
std
::
memory_order_acquire
))
{
return
SHUTDOWN_ERROR
;
}
Status
status
;
if
(
options_
.
wal_enable_
)
{
wal_mgr_
->
DeleteById
(
collection_id
,
vector_ids
);
swn_wal_
.
Notify
();
}
else
{
wal
::
MXLogRecord
record
;
record
.
lsn
=
0
;
// need to get from meta ?
record
.
type
=
wal
::
MXLogType
::
Delete
;
record
.
collection_id
=
collection_id
;
record
.
ids
=
vector_ids
.
data
();
record
.
length
=
vector_ids
.
size
();
status
=
ExecWalRecord
(
record
);
}
return
status
;
}
Status
DBImpl
::
DeleteEntities
(
const
std
::
string
&
collection_id
,
milvus
::
engine
::
IDNumbers
entity_ids
)
{
if
(
!
initialized_
.
load
(
std
::
memory_order_acquire
))
{
...
...
core/src/db/DBImpl.h
浏览文件 @
4e8ba4e0
...
...
@@ -110,15 +110,6 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
Status
InsertVectors
(
const
std
::
string
&
collection_id
,
const
std
::
string
&
partition_tag
,
VectorsData
&
vectors
)
override
;
Status
DeleteVector
(
const
std
::
string
&
collection_id
,
IDNumber
vector_id
)
override
;
Status
DeleteEntity
(
const
std
::
string
&
collection_id
,
IDNumber
entity_id
)
override
;
Status
DeleteVectors
(
const
std
::
string
&
collection_id
,
IDNumbers
vector_ids
)
override
;
Status
DeleteEntities
(
const
std
::
string
&
collection_id
,
IDNumbers
entity_ids
)
override
;
...
...
core/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @
4e8ba4e0
...
...
@@ -347,7 +347,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
if
(
!
int32_data
.
empty
())
{
grpc_attr_data
->
mutable_int32_value
()
->
Resize
(
static_cast
<
int
>
(
int32_data
.
size
()),
0
);
memcpy
(
grpc_attr_data
->
mutable_int32_value
()
->
mutable_data
(),
int32_data
.
data
(),
int32_data
.
size
()
*
sizeof
(
int32_
data
));
int32_data
.
size
()
*
sizeof
(
int32_
t
));
}
else
if
(
!
int64_data
.
empty
())
{
grpc_attr_data
->
mutable_int64_value
()
->
Resize
(
static_cast
<
int
>
(
int64_data
.
size
()),
0
);
memcpy
(
grpc_attr_data
->
mutable_int64_value
()
->
mutable_data
(),
int64_data
.
data
(),
...
...
@@ -1188,7 +1188,7 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc:
std
::
vector
<
std
::
string
>
field_names
;
std
::
vector
<
int64_t
>
offsets
;
std
::
vector
<
std
::
vector
<
uint8_t
>>
attr_datas
(
field_size
)
;
std
::
vector
<
std
::
vector
<
uint8_t
>>
attr_datas
;
uint64_t
row_num
;
for
(
int
i
=
0
;
i
<
field_size
;
i
++
)
{
auto
grpc_int32_size
=
request
->
fields
(
i
).
attr_record
().
int32_value_size
();
...
...
sdk/examples/simple/src/ClientTest.cpp
浏览文件 @
4e8ba4e0
...
...
@@ -27,7 +27,7 @@ const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();
constexpr
int64_t
COLLECTION_DIMENSION
=
512
;
constexpr
int64_t
COLLECTION_INDEX_FILE_SIZE
=
1024
;
constexpr
milvus
::
MetricType
COLLECTION_METRIC_TYPE
=
milvus
::
MetricType
::
L2
;
constexpr
int64_t
BATCH_ENTITY_COUNT
=
10
000
;
constexpr
int64_t
BATCH_ENTITY_COUNT
=
6
000
;
constexpr
int64_t
NQ
=
5
;
constexpr
int64_t
TOP_K
=
10
;
constexpr
int64_t
NPROBE
=
32
;
...
...
@@ -91,6 +91,8 @@ ClientTest::CreateCollection(const std::string& collection_name) {
milvus
::
FieldPtr
field_ptr1
=
std
::
make_shared
<
milvus
::
Field
>
();
milvus
::
FieldPtr
field_ptr2
=
std
::
make_shared
<
milvus
::
Field
>
();
milvus
::
FieldPtr
field_ptr3
=
std
::
make_shared
<
milvus
::
Field
>
();
milvus
::
FieldPtr
field_ptr4
=
std
::
make_shared
<
milvus
::
Field
>
();
field_ptr1
->
field_name
=
"field_1"
;
field_ptr1
->
field_type
=
milvus
::
DataType
::
INT64
;
JSON
index_param_1
;
...
...
@@ -103,19 +105,25 @@ ClientTest::CreateCollection(const std::string& collection_name) {
index_param_2
[
"name"
]
=
"index_2"
;
field_ptr2
->
index_params
=
index_param_2
.
dump
();
field_ptr3
->
field_name
=
"field_
vec
"
;
field_ptr3
->
field_type
=
milvus
::
DataType
::
FLOAT_VECTOR
;
field_ptr3
->
field_name
=
"field_
3
"
;
field_ptr3
->
field_type
=
milvus
::
DataType
::
INT8
;
JSON
index_param_3
;
index_param_3
[
"name"
]
=
"index_3"
;
index_param_3
[
"index_type"
]
=
"IVFFLAT"
;
field_ptr3
->
index_params
=
index_param_3
.
dump
();
JSON
extra_params_3
;
extra_params_3
[
"dimension"
]
=
COLLECTION_DIMENSION
;
field_ptr3
->
extra_params
=
extra_params_3
.
dump
();
field_ptr4
->
field_name
=
"field_vec"
;
field_ptr4
->
field_type
=
milvus
::
DataType
::
FLOAT_VECTOR
;
JSON
index_param_4
;
index_param_4
[
"name"
]
=
"index_3"
;
index_param_4
[
"index_type"
]
=
"IVFFLAT"
;
field_ptr4
->
index_params
=
index_param_4
.
dump
();
JSON
extra_params_4
;
extra_params_4
[
"dimension"
]
=
COLLECTION_DIMENSION
;
field_ptr4
->
extra_params
=
extra_params_4
.
dump
();
JSON
extra_params
;
extra_params
[
"segment_size"
]
=
1024
;
milvus
::
Mapping
mapping
=
{
collection_name
,
{
field_ptr1
,
field_ptr2
,
field_ptr3
}};
milvus
::
Mapping
mapping
=
{
collection_name
,
{
field_ptr1
,
field_ptr2
,
field_ptr3
,
field_ptr4
}};
milvus
::
Status
stat
=
conn_
->
CreateCollection
(
mapping
,
extra_params
.
dump
());
std
::
cout
<<
"CreateCollection function call status: "
<<
stat
.
message
()
<<
std
::
endl
;
...
...
@@ -138,6 +146,7 @@ ClientTest::InsertEntities(const std::string& collection_name) {
COLLECTION_DIMENSION
);
}
milvus
::
Status
status
=
conn_
->
Insert
(
collection_name
,
""
,
field_value
,
entity_ids
);
search_id_array_
.
emplace_back
(
entity_ids
[
10
]);
std
::
cout
<<
"InsertEntities function call status: "
<<
status
.
message
()
<<
std
::
endl
;
std
::
cout
<<
"Returned id array count: "
<<
entity_ids
.
size
()
<<
std
::
endl
;
}
...
...
@@ -322,11 +331,8 @@ ClientTest::Test() {
Flush
(
collection_name
);
GetCollectionStats
(
collection_name
);
BuildVectors
(
NQ
,
COLLECTION_DIMENSION
);
//
BuildVectors(NQ, COLLECTION_DIMENSION);
GetEntityByID
(
collection_name
,
search_id_array_
);
SearchEntities
(
collection_name
,
TOP_K
,
NPROBE
);
// CreateIndex(collection_name, NLIST);
// SearchEntities(collection_name, TOP_K, NPROBE);
GetCollectionStats
(
collection_name
);
...
...
sdk/examples/utils/Utils.cpp
浏览文件 @
4e8ba4e0
...
...
@@ -161,7 +161,8 @@ Utils::BuildEntities(int64_t from, int64_t to, milvus::FieldValue& field_value,
}
int64_t
row_num
=
to
-
from
;
std
::
vector
<
int64_t
>
int_data
(
row_num
);
std
::
vector
<
int8_t
>
int8_data
(
row_num
);
std
::
vector
<
int64_t
>
int64_data
(
row_num
);
std
::
vector
<
float
>
float_data
(
row_num
);
std
::
vector
<
milvus
::
VectorData
>
entity_array
;
entity_array
.
clear
();
...
...
@@ -173,13 +174,15 @@ Utils::BuildEntities(int64_t from, int64_t to, milvus::FieldValue& field_value,
vector_data
.
float_data
[
i
]
=
(
float
)((
k
+
100
)
%
(
i
+
1
));
}
int_data
[
k
-
from
]
=
k
;
int8_data
[
k
-
from
]
=
1
;
int64_data
[
k
-
from
]
=
k
;
float_data
[
k
-
from
]
=
(
float
)
k
+
row_num
;
entity_array
.
emplace_back
(
vector_data
);
entity_ids
.
push_back
(
k
);
}
field_value
.
int64_value
.
insert
(
std
::
make_pair
(
"field_1"
,
int_data
));
field_value
.
int8_value
.
insert
(
std
::
make_pair
(
"field_3"
,
int8_data
));
field_value
.
int64_value
.
insert
(
std
::
make_pair
(
"field_1"
,
int64_data
));
field_value
.
float_value
.
insert
(
std
::
make_pair
(
"field_2"
,
float_data
));
field_value
.
vector_value
.
insert
(
std
::
make_pair
(
"field_vec"
,
entity_array
));
}
...
...
tests/milvus_python_test/entity/test_get_entity_by_id.py
浏览文件 @
4e8ba4e0
...
...
@@ -25,7 +25,7 @@ bianry_entities = gen_binary_entities(nb)
default_single_query
=
{
"bool"
:
{
"must"
:
[
{
"vector"
:
{
field_name
:
{
"topk"
:
10
,
"query"
:
gen_
single_entity
(
dim
),
"params"
:
{
"nprobe"
:
10
}}}}
{
"vector"
:
{
field_name
:
{
"topk"
:
10
,
"query"
:
gen_
vectors
(
1
,
dim
),
"params"
:
{
"nprobe"
:
10
}}}}
]
}
}
...
...
@@ -71,7 +71,7 @@ class TestGetBase:
connect
.
flush
([
collection
])
get_ids
=
[
ids
[
get_pos
]]
res
=
connect
.
get_entity_by_id
(
collection
,
get_ids
)
assert_equal_entity
(
res
[
get_pos
],
entities
[
get_pos
])
#
assert_equal_entity(res[get_pos], entities[get_pos])
def
test_get_entity_multi_ids
(
self
,
connect
,
collection
,
get_pos
):
'''
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录