Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7f044fff
M
milvus
项目概览
milvus
/
milvus
10 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
7f044fff
编写于
1月 21, 2021
作者:
F
FluorineDog
提交者:
yefu.chen
1月 21, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Enable search for sealed
Signed-off-by:
N
FluorineDog
<
guilin.gou@zilliz.com
>
上级
be677567
变更
34
隐藏空白更改
内联
并排
Showing
34 changed file
with
208 addition
and
181 deletion
+208
-181
docs/developer_guides/chap01_system_overview.md
docs/developer_guides/chap01_system_overview.md
+3
-3
internal/core/src/query/Plan.cpp
internal/core/src/query/Plan.cpp
+1
-1
internal/core/src/query/PlanNode.h
internal/core/src/query/PlanNode.h
+2
-1
internal/core/src/query/SearchOnGrowing.cpp
internal/core/src/query/SearchOnGrowing.cpp
+11
-24
internal/core/src/query/SearchOnGrowing.h
internal/core/src/query/SearchOnGrowing.h
+1
-2
internal/core/src/query/SearchOnSealed.cpp
internal/core/src/query/SearchOnSealed.cpp
+1
-3
internal/core/src/query/generated/ExecExprVisitor.h
internal/core/src/query/generated/ExecExprVisitor.h
+1
-1
internal/core/src/query/generated/ExecPlanNodeVisitor.h
internal/core/src/query/generated/ExecPlanNodeVisitor.h
+2
-2
internal/core/src/query/visitors/ExecExprVisitor.cpp
internal/core/src/query/visitors/ExecExprVisitor.cpp
+1
-1
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
+5
-10
internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
+12
-12
internal/core/src/segcore/SegmentGrowingImpl.cpp
internal/core/src/segcore/SegmentGrowingImpl.cpp
+15
-11
internal/core/src/segcore/SegmentGrowingImpl.h
internal/core/src/segcore/SegmentGrowingImpl.h
+8
-6
internal/core/src/segcore/SegmentInterface.cpp
internal/core/src/segcore/SegmentInterface.cpp
+13
-0
internal/core/src/segcore/SegmentInterface.h
internal/core/src/segcore/SegmentInterface.h
+11
-2
internal/core/src/segcore/SegmentSealedImpl.cpp
internal/core/src/segcore/SegmentSealedImpl.cpp
+15
-8
internal/core/src/segcore/SegmentSealedImpl.h
internal/core/src/segcore/SegmentSealedImpl.h
+8
-6
internal/core/src/segcore/plan_c.cpp
internal/core/src/segcore/plan_c.cpp
+2
-1
internal/core/src/segcore/segment_c.cpp
internal/core/src/segcore/segment_c.cpp
+2
-1
internal/core/unittest/test_expr.cpp
internal/core/unittest/test_expr.cpp
+2
-1
internal/core/unittest/test_query.cpp
internal/core/unittest/test_query.cpp
+3
-41
internal/core/unittest/test_sealed.cpp
internal/core/unittest/test_sealed.cpp
+39
-2
internal/core/unittest/test_utils/DataGen.cpp
internal/core/unittest/test_utils/DataGen.cpp
+14
-0
internal/distributed/datanode/client.go
internal/distributed/datanode/client.go
+3
-3
internal/distributed/indexservice/service.go
internal/distributed/indexservice/service.go
+2
-4
internal/distributed/querynode/client.go
internal/distributed/querynode/client.go
+3
-3
internal/distributed/queryservice/client.go
internal/distributed/queryservice/client.go
+3
-3
internal/distributed/queryservice/service.go
internal/distributed/queryservice/service.go
+3
-3
internal/indexnode/indexnode.go
internal/indexnode/indexnode.go
+3
-3
internal/indexservice/indexservice.go
internal/indexservice/indexservice.go
+3
-3
internal/proxyservice/proxyservice.go
internal/proxyservice/proxyservice.go
+3
-3
internal/querynode/query_node.go
internal/querynode/query_node.go
+7
-11
internal/queryservice/queryservice.go
internal/queryservice/queryservice.go
+3
-3
internal/util/typeutil/service.go
internal/util/typeutil/service.go
+3
-3
未找到文件。
docs/developer_guides/chap01_system_overview.md
浏览文件 @
7f044fff
...
...
@@ -78,9 +78,9 @@ In order to boost throughput, we model Milvus as a stream-driven system.
```
go
type
Component
interface
{
Init
()
error
Start
()
error
Stop
()
error
Init
()
Start
()
Stop
()
GetComponentStates
()
(
ComponentStates
,
error
)
GetTimeTickChannel
()
(
string
,
error
)
GetStatisticsChannel
()
(
string
,
error
)
...
...
internal/core/src/query/Plan.cpp
浏览文件 @
7f044fff
...
...
@@ -209,7 +209,7 @@ Parser::ParseVecNode(const Json& out_body) {
}
}();
vec_node
->
query_info_
.
topK_
=
topK
;
vec_node
->
query_info_
.
metric_type_
=
vec_info
.
at
(
"metric_type"
);
vec_node
->
query_info_
.
metric_type_
=
GetMetricType
(
vec_info
.
at
(
"metric_type"
)
);
vec_node
->
query_info_
.
search_params_
=
vec_info
.
at
(
"params"
);
vec_node
->
query_info_
.
field_offset_
=
field_offset
;
vec_node
->
placeholder_tag_
=
vec_info
.
at
(
"query"
);
...
...
internal/core/src/query/PlanNode.h
浏览文件 @
7f044fff
...
...
@@ -41,7 +41,8 @@ using PlanNodePtr = std::unique_ptr<PlanNode>;
struct
QueryInfo
{
int64_t
topK_
;
FieldOffset
field_offset_
;
std
::
string
metric_type_
;
// TODO: use enum
MetricType
metric_type_
;
std
::
string
deprecated_metric_type_
;
// TODO: use enum
nlohmann
::
json
search_params_
;
};
...
...
internal/core/src/query/SearchOnGrowing.cpp
浏览文件 @
7f044fff
...
...
@@ -63,7 +63,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
auto
dim
=
field
.
get_dim
();
auto
topK
=
info
.
topK_
;
auto
total_count
=
topK
*
num_queries
;
auto
metric_type
=
GetMetricType
(
info
.
metric_type_
)
;
auto
metric_type
=
info
.
metric_type_
;
// step 3: small indexing search
// std::vector<int64_t> final_uids(total_count, -1);
...
...
@@ -138,7 +138,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
auto
&
record
=
segment
.
get_insert_record
();
// step 1: binary search to find the barrier of the snapshot
// auto ins_barrier = get_barrier(record, timestamp);
auto
metric_type
=
GetMetricType
(
info
.
metric_type_
)
;
auto
metric_type
=
info
.
metric_type_
;
// auto del_barrier = get_barrier(deleted_record_, timestamp);
#if 0
...
...
@@ -195,37 +195,24 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
}
// TODO: refactor and merge this into one
template
<
typename
VectorType
>
void
SearchOnGrowing
(
const
segcore
::
SegmentGrowingImpl
&
segment
,
int64_t
ins_barrier
,
const
query
::
QueryInfo
&
info
,
const
EmbeddedType
<
VectorType
>
*
query_data
,
const
void
*
query_data
,
int64_t
num_queries
,
const
faiss
::
BitsetView
&
bitset
,
QueryResult
&
results
)
{
static_assert
(
IsVector
<
VectorType
>
);
if
constexpr
(
std
::
is_same_v
<
VectorType
,
FloatVector
>
)
{
FloatSearch
(
segment
,
info
,
query_data
,
num_queries
,
ins_barrier
,
bitset
,
results
);
// TODO: add data_type to info
auto
data_type
=
segment
.
get_schema
()[
info
.
field_offset_
].
get_data_type
();
Assert
(
datatype_is_vector
(
data_type
));
if
(
data_type
==
DataType
::
VECTOR_FLOAT
)
{
auto
typed_data
=
reinterpret_cast
<
const
float
*>
(
query_data
);
FloatSearch
(
segment
,
info
,
typed_data
,
num_queries
,
ins_barrier
,
bitset
,
results
);
}
else
{
BinarySearch
(
segment
,
info
,
query_data
,
num_queries
,
ins_barrier
,
bitset
,
results
);
auto
typed_data
=
reinterpret_cast
<
const
uint8_t
*>
(
query_data
);
BinarySearch
(
segment
,
info
,
typed_data
,
num_queries
,
ins_barrier
,
bitset
,
results
);
}
}
template
void
SearchOnGrowing
<
FloatVector
>(
const
segcore
::
SegmentGrowingImpl
&
segment
,
int64_t
ins_barrier
,
const
query
::
QueryInfo
&
info
,
const
EmbeddedType
<
FloatVector
>*
query_data
,
int64_t
num_queries
,
const
faiss
::
BitsetView
&
bitset
,
QueryResult
&
results
);
template
void
SearchOnGrowing
<
BinaryVector
>(
const
segcore
::
SegmentGrowingImpl
&
segment
,
int64_t
ins_barrier
,
const
query
::
QueryInfo
&
info
,
const
EmbeddedType
<
BinaryVector
>*
query_data
,
int64_t
num_queries
,
const
faiss
::
BitsetView
&
bitset
,
QueryResult
&
results
);
}
// namespace milvus::query
internal/core/src/query/SearchOnGrowing.h
浏览文件 @
7f044fff
...
...
@@ -20,12 +20,11 @@ namespace milvus::query {
using
BitmapChunk
=
boost
::
dynamic_bitset
<>
;
using
BitmapSimple
=
std
::
deque
<
BitmapChunk
>
;
template
<
typename
VectorType
>
void
SearchOnGrowing
(
const
segcore
::
SegmentGrowingImpl
&
segment
,
int64_t
ins_barrier
,
const
query
::
QueryInfo
&
info
,
const
EmbeddedType
<
VectorType
>
*
query_data
,
const
void
*
query_data
,
int64_t
num_queries
,
const
faiss
::
BitsetView
&
bitset
,
QueryResult
&
results
);
...
...
internal/core/src/query/SearchOnSealed.cpp
浏览文件 @
7f044fff
...
...
@@ -63,9 +63,7 @@ SearchOnSealed(const Schema& schema,
Assert
(
record
.
is_ready
(
field_offset
));
auto
indexing_entry
=
record
.
get_entry
(
field_offset
);
std
::
cout
<<
" SearchOnSealed, indexing_entry->metric:"
<<
indexing_entry
->
metric_type_
<<
std
::
endl
;
std
::
cout
<<
" SearchOnSealed, query_info.metric_type_:"
<<
query_info
.
metric_type_
<<
std
::
endl
;
Assert
(
indexing_entry
->
metric_type_
==
GetMetricType
(
query_info
.
metric_type_
));
Assert
(
indexing_entry
->
metric_type_
==
query_info
.
metric_type_
);
auto
final
=
[
&
]
{
auto
ds
=
knowhere
::
GenDataset
(
num_queries
,
dim
,
query_data
);
...
...
internal/core/src/query/generated/ExecExprVisitor.h
浏览文件 @
7f044fff
...
...
@@ -37,7 +37,7 @@ class ExecExprVisitor : public ExprVisitor {
public:
using
RetType
=
std
::
deque
<
boost
::
dynamic_bitset
<>>
;
ExecExprVisitor
(
const
segcore
::
Segment
GrowingImpl
&
segment
,
int64_t
row_count
)
ExecExprVisitor
(
const
segcore
::
Segment
InternalInterface
&
segment
,
int64_t
row_count
)
:
segment_
(
segment
),
row_count_
(
row_count
)
{
}
RetType
...
...
internal/core/src/query/generated/ExecPlanNodeVisitor.h
浏览文件 @
7f044fff
...
...
@@ -29,7 +29,7 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
public:
using
RetType
=
QueryResult
;
ExecPlanNodeVisitor
(
const
segcore
::
Segment
Growing
&
segment
,
ExecPlanNodeVisitor
(
const
segcore
::
Segment
Interface
&
segment
,
Timestamp
timestamp
,
const
PlaceholderGroup
&
placeholder_group
)
:
segment_
(
segment
),
timestamp_
(
timestamp
),
placeholder_group_
(
placeholder_group
)
{
...
...
@@ -53,7 +53,7 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
private:
// std::optional<RetType> ret_;
const
segcore
::
Segment
Growing
&
segment_
;
const
segcore
::
Segment
Interface
&
segment_
;
Timestamp
timestamp_
;
const
PlaceholderGroup
&
placeholder_group_
;
...
...
internal/core/src/query/visitors/ExecExprVisitor.cpp
浏览文件 @
7f044fff
...
...
@@ -25,7 +25,7 @@ namespace impl {
class
ExecExprVisitor
:
ExprVisitor
{
public:
using
RetType
=
std
::
deque
<
boost
::
dynamic_bitset
<>>
;
ExecExprVisitor
(
const
segcore
::
Segment
GrowingImpl
&
segment
,
int64_t
row_count
)
ExecExprVisitor
(
const
segcore
::
Segment
InternalInterface
&
segment
,
int64_t
row_count
)
:
segment_
(
segment
),
row_count_
(
row_count
)
{
}
RetType
...
...
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
浏览文件 @
7f044fff
...
...
@@ -28,7 +28,7 @@ namespace impl {
class
ExecPlanNodeVisitor
:
PlanNodeVisitor
{
public:
using
RetType
=
QueryResult
;
ExecPlanNodeVisitor
(
const
segcore
::
Segment
Growing
&
segment
,
ExecPlanNodeVisitor
(
const
segcore
::
Segment
Interface
&
segment
,
Timestamp
timestamp
,
const
PlaceholderGroup
&
placeholder_group
)
:
segment_
(
segment
),
timestamp_
(
timestamp
),
placeholder_group_
(
placeholder_group
)
{
...
...
@@ -52,7 +52,7 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
private:
// std::optional<RetType> ret_;
const
segcore
::
Segment
Growing
&
segment_
;
const
segcore
::
Segment
Interface
&
segment_
;
Timestamp
timestamp_
;
const
PlaceholderGroup
&
placeholder_group_
;
...
...
@@ -66,7 +66,7 @@ void
ExecPlanNodeVisitor
::
VectorVisitorImpl
(
VectorPlanNode
&
node
)
{
// TODO: optimize here, remove the dynamic cast
assert
(
!
ret_
.
has_value
());
auto
segment
=
dynamic_cast
<
const
segcore
::
Segment
GrowingImpl
*>
(
&
segment_
);
auto
segment
=
dynamic_cast
<
const
segcore
::
Segment
InternalInterface
*>
(
&
segment_
);
AssertInfo
(
segment
,
"support SegmentSmallIndex Only"
);
RetType
ret
;
auto
&
ph
=
placeholder_group_
.
at
(
0
);
...
...
@@ -76,7 +76,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
aligned_vector
<
uint8_t
>
bitset_holder
;
BitsetView
view
;
// TODO: add API to unify row_count
auto
row_count
=
seg
core
::
get_barrier
(
segment
->
get_insert_record
(),
timestamp_
);
auto
row_count
=
seg
ment
->
get_row_count
(
);
if
(
node
.
predicate_
.
has_value
())
{
ExecExprVisitor
::
RetType
expr_ret
=
ExecExprVisitor
(
*
segment
,
row_count
).
call_child
(
*
node
.
predicate_
.
value
());
...
...
@@ -84,12 +84,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
view
=
BitsetView
(
bitset_holder
.
data
(),
bitset_holder
.
size
()
*
8
);
}
auto
&
sealed_indexing
=
segment
->
get_sealed_indexing_record
();
if
(
sealed_indexing
.
is_ready
(
node
.
query_info_
.
field_offset_
))
{
SearchOnSealed
(
segment
->
get_schema
(),
sealed_indexing
,
node
.
query_info_
,
src_data
,
num_queries
,
view
,
ret
);
}
else
{
SearchOnGrowing
<
VectorType
>
(
*
segment
,
row_count
,
node
.
query_info_
,
src_data
,
num_queries
,
view
,
ret
);
}
segment
->
vector_search
(
row_count
,
node
.
query_info_
,
src_data
,
num_queries
,
view
,
ret
);
ret_
=
ret
;
}
...
...
internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
浏览文件 @
7f044fff
...
...
@@ -54,12 +54,12 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
assert
(
!
ret_
);
auto
&
info
=
node
.
query_info_
;
Json
json_body
{
{
"node_type"
,
"FloatVectorANNS"
},
//
{
"metric_type"
,
info
.
metric_type_
},
//
{
"field_offset_"
,
info
.
field_offset_
.
get
()},
//
{
"topK"
,
info
.
topK_
},
//
{
"search_params"
,
info
.
search_params_
},
//
{
"placeholder_tag"
,
node
.
placeholder_tag_
},
//
{
"node_type"
,
"FloatVectorANNS"
},
//
{
"metric_type"
,
MetricTypeToName
(
info
.
metric_type_
)},
//
{
"field_offset_"
,
info
.
field_offset_
.
get
()},
//
{
"topK"
,
info
.
topK_
},
//
{
"search_params"
,
info
.
search_params_
},
//
{
"placeholder_tag"
,
node
.
placeholder_tag_
},
//
};
if
(
node
.
predicate_
.
has_value
())
{
ShowExprVisitor
expr_show
;
...
...
@@ -76,12 +76,12 @@ ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
assert
(
!
ret_
);
auto
&
info
=
node
.
query_info_
;
Json
json_body
{
{
"node_type"
,
"BinaryVectorANNS"
},
//
{
"metric_type"
,
info
.
metric_type_
},
//
{
"field_offset_"
,
info
.
field_offset_
.
get
()},
//
{
"topK"
,
info
.
topK_
},
//
{
"search_params"
,
info
.
search_params_
},
//
{
"placeholder_tag"
,
node
.
placeholder_tag_
},
//
{
"node_type"
,
"BinaryVectorANNS"
},
//
{
"metric_type"
,
MetricTypeToName
(
info
.
metric_type_
)},
//
{
"field_offset_"
,
info
.
field_offset_
.
get
()},
//
{
"topK"
,
info
.
topK_
},
//
{
"search_params"
,
info
.
search_params_
},
//
{
"placeholder_tag"
,
node
.
placeholder_tag_
},
//
};
if
(
node
.
predicate_
.
has_value
())
{
ShowExprVisitor
expr_show
;
...
...
internal/core/src/segcore/SegmentGrowingImpl.cpp
浏览文件 @
7f044fff
...
...
@@ -19,6 +19,7 @@
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
#include <knowhere/index/vector_index/VecIndexFactory.h>
#include <faiss/utils/distances.h>
#include <query/SearchOnSealed.h>
#include "query/generated/ExecPlanNodeVisitor.h"
#include "segcore/SegmentGrowingImpl.h"
#include "query/PlanNode.h"
...
...
@@ -237,17 +238,6 @@ SegmentGrowingImpl::GetMemoryUsageInBytes() const {
return
total_bytes
;
}
QueryResult
SegmentGrowingImpl
::
Search
(
const
query
::
Plan
*
plan
,
const
query
::
PlaceholderGroup
**
placeholder_groups
,
const
Timestamp
*
timestamps
,
int64_t
num_groups
)
const
{
Assert
(
num_groups
==
1
);
query
::
ExecPlanNodeVisitor
visitor
(
*
this
,
timestamps
[
0
],
*
placeholder_groups
[
0
]);
auto
results
=
visitor
.
get_moved_result
(
*
plan
->
plan_node_
);
return
results
;
}
Status
SegmentGrowingImpl
::
LoadIndexing
(
const
LoadIndexInfo
&
info
)
{
auto
field_offset
=
schema_
->
get_offset
(
FieldName
(
info
.
field_name
));
...
...
@@ -270,5 +260,19 @@ SegmentGrowingImpl::num_chunk_data() const {
auto
size
=
get_insert_record
().
ack_responder_
.
GetAck
();
return
upper_div
(
size
,
chunk_size_
);
}
void
SegmentGrowingImpl
::
vector_search
(
int64_t
vec_count
,
query
::
QueryInfo
query_info
,
const
void
*
query_data
,
int64_t
query_count
,
const
BitsetView
&
bitset
,
QueryResult
&
output
)
const
{
auto
&
sealed_indexing
=
this
->
get_sealed_indexing_record
();
if
(
sealed_indexing
.
is_ready
(
query_info
.
field_offset_
))
{
query
::
SearchOnSealed
(
this
->
get_schema
(),
sealed_indexing
,
query_info
,
query_data
,
query_count
,
bitset
,
output
);
}
else
{
SearchOnGrowing
(
*
this
,
vec_count
,
query_info
,
query_data
,
query_count
,
bitset
,
output
);
}
}
}
// namespace milvus::segcore
internal/core/src/segcore/SegmentGrowingImpl.h
浏览文件 @
7f044fff
...
...
@@ -53,12 +53,6 @@ class SegmentGrowingImpl : public SegmentGrowing {
Status
Delete
(
int64_t
reserverd_offset
,
int64_t
size
,
const
int64_t
*
row_ids
,
const
Timestamp
*
timestamps
)
override
;
QueryResult
Search
(
const
query
::
Plan
*
Plan
,
const
query
::
PlaceholderGroup
*
placeholder_groups
[],
const
Timestamp
timestamps
[],
int64_t
num_groups
)
const
override
;
// stop receive insert requests
// will move data to immutable vector or something
Status
...
...
@@ -181,6 +175,14 @@ class SegmentGrowingImpl : public SegmentGrowing {
indexing_record_
(
*
schema_
,
chunk_size
)
{
}
void
vector_search
(
int64_t
vec_count
,
query
::
QueryInfo
query_info
,
const
void
*
query_data
,
int64_t
query_count
,
const
BitsetView
&
bitset
,
QueryResult
&
output
)
const
override
;
public:
std
::
shared_ptr
<
DeletedRecord
::
TmpBitmap
>
get_deleted_bitmap
(
int64_t
del_barrier
,
Timestamp
query_timestamp
,
int64_t
insert_barrier
,
bool
force
=
false
);
...
...
internal/core/src/segcore/SegmentInterface.cpp
浏览文件 @
7f044fff
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "segcore/SegmentInterface.h"
#include "query/generated/ExecPlanNodeVisitor.h"
namespace
milvus
::
segcore
{
class
Naive
;
...
...
@@ -39,4 +40,16 @@ SegmentInterface::FillTargetEntry(const query::Plan* plan, QueryResult& results)
results
.
row_data_
.
emplace_back
(
std
::
move
(
blob
));
}
}
QueryResult
SegmentInterface
::
Search
(
const
query
::
Plan
*
plan
,
const
query
::
PlaceholderGroup
**
placeholder_groups
,
const
Timestamp
*
timestamps
,
int64_t
num_groups
)
const
{
Assert
(
num_groups
==
1
);
query
::
ExecPlanNodeVisitor
visitor
(
*
this
,
timestamps
[
0
],
*
placeholder_groups
[
0
]);
auto
results
=
visitor
.
get_moved_result
(
*
plan
->
plan_node_
);
return
results
;
}
}
// namespace milvus::segcore
internal/core/src/segcore/SegmentInterface.h
浏览文件 @
7f044fff
...
...
@@ -17,6 +17,7 @@
#include "IndexingEntry.h"
#include <knowhere/index/vector_index/VecIndex.h>
#include "common/SystemProperty.h"
#include "query/PlanNode.h"
namespace
milvus
::
segcore
{
...
...
@@ -25,11 +26,11 @@ class SegmentInterface {
void
FillTargetEntry
(
const
query
::
Plan
*
plan
,
QueryResult
&
results
)
const
;
virtual
QueryResult
QueryResult
Search
(
const
query
::
Plan
*
Plan
,
const
query
::
PlaceholderGroup
*
placeholder_groups
[],
const
Timestamp
timestamps
[],
int64_t
num_groups
)
const
=
0
;
int64_t
num_groups
)
const
;
virtual
int64_t
GetMemoryUsageInBytes
()
const
=
0
;
...
...
@@ -71,6 +72,14 @@ class SegmentInternalInterface : public SegmentInterface {
}
public:
virtual
void
vector_search
(
int64_t
vec_count
,
query
::
QueryInfo
query_info
,
const
void
*
query_data
,
int64_t
query_count
,
const
BitsetView
&
bitset
,
QueryResult
&
output
)
const
=
0
;
virtual
int64_t
num_chunk_index_safe
(
FieldOffset
field_offset
)
const
=
0
;
...
...
internal/core/src/segcore/SegmentSealedImpl.cpp
浏览文件 @
7f044fff
...
...
@@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "segcore/SegmentSealedImpl.h"
#include "query/SearchOnSealed.h"
namespace
milvus
::
segcore
{
void
SegmentSealedImpl
::
LoadIndex
(
const
LoadIndexInfo
&
info
)
{
...
...
@@ -107,14 +108,6 @@ SegmentSealedImpl::chunk_index_impl(FieldOffset field_offset, int64_t chunk_id)
return
nullptr
;
}
QueryResult
SegmentSealedImpl
::
Search
(
const
query
::
Plan
*
Plan
,
const
query
::
PlaceholderGroup
**
placeholder_groups
,
const
Timestamp
*
timestamps
,
int64_t
num_groups
)
const
{
PanicInfo
(
"unimplemented"
);
}
int64_t
SegmentSealedImpl
::
GetMemoryUsageInBytes
()
const
{
// TODO: add estimate for index
...
...
@@ -134,6 +127,20 @@ SegmentSealedImpl::get_schema() const {
return
*
schema_
;
}
void
SegmentSealedImpl
::
vector_search
(
int64_t
vec_count
,
query
::
QueryInfo
query_info
,
const
void
*
query_data
,
int64_t
query_count
,
const
BitsetView
&
bitset
,
QueryResult
&
output
)
const
{
auto
field_offset
=
query_info
.
field_offset_
;
auto
&
field_meta
=
schema_
->
operator
[](
field_offset
);
Assert
(
field_meta
.
is_vector
());
Assert
(
vec_indexings_
.
is_ready
(
field_offset
));
query
::
SearchOnSealed
(
*
schema_
,
vec_indexings_
,
query_info
,
query_data
,
query_count
,
bitset
,
output
);
}
SegmentSealedPtr
CreateSealedSegment
(
SchemaPtr
schema
,
int64_t
chunk_size
)
{
return
std
::
make_unique
<
SegmentSealedImpl
>
(
schema
);
...
...
internal/core/src/segcore/SegmentSealedImpl.h
浏览文件 @
7f044fff
...
...
@@ -26,12 +26,6 @@ class SegmentSealedImpl : public SegmentSealed {
LoadFieldData
(
const
LoadFieldDataInfo
&
info
)
override
;
public:
QueryResult
Search
(
const
query
::
Plan
*
Plan
,
const
query
::
PlaceholderGroup
*
placeholder_groups
[],
const
Timestamp
timestamps
[],
int64_t
num_groups
)
const
override
;
int64_t
GetMemoryUsageInBytes
()
const
override
;
...
...
@@ -104,6 +98,14 @@ class SegmentSealedImpl : public SegmentSealed {
}
}
void
vector_search
(
int64_t
vec_count
,
query
::
QueryInfo
query_info
,
const
void
*
query_data
,
int64_t
query_count
,
const
BitsetView
&
bitset
,
QueryResult
&
output
)
const
override
;
bool
is_all_ready
()
const
{
// TODO: optimize here
...
...
internal/core/src/segcore/plan_c.cpp
浏览文件 @
7f044fff
...
...
@@ -78,7 +78,8 @@ GetTopK(CPlan plan) {
const
char
*
GetMetricType
(
CPlan
plan
)
{
auto
query_plan
=
static_cast
<
milvus
::
query
::
Plan
*>
(
plan
);
return
strdup
(
query_plan
->
plan_node_
->
query_info_
.
metric_type_
.
c_str
());
auto
metric_str
=
milvus
::
MetricTypeToName
(
query_plan
->
plan_node_
->
query_info_
.
metric_type_
);
return
strdup
(
metric_str
.
c_str
());
}
void
...
...
internal/core/src/segcore/segment_c.cpp
浏览文件 @
7f044fff
...
...
@@ -20,6 +20,7 @@
#include "common/type_c.h"
#include <knowhere/index/vector_index/VecIndex.h>
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
#include "common/Types.h"
////////////////////////////// common interfaces //////////////////////////////
CSegmentInterface
...
...
@@ -79,7 +80,7 @@ Search(CSegmentInterface c_segment,
auto
status
=
CStatus
();
try
{
*
query_result
=
segment
->
Search
(
plan
,
placeholder_groups
.
data
(),
timestamps
,
num_groups
);
if
(
plan
->
plan_node_
->
query_info_
.
metric_type_
!=
"IP"
)
{
if
(
plan
->
plan_node_
->
query_info_
.
metric_type_
!=
milvus
::
MetricType
::
METRIC_INNER_PRODUCT
)
{
for
(
auto
&
dis
:
query_result
->
result_distances_
)
{
dis
*=
-
1
;
}
...
...
internal/core/unittest/test_expr.cpp
浏览文件 @
7f044fff
...
...
@@ -235,7 +235,8 @@ TEST(Expr, ShowExecutor) {
int64_t
num_queries
=
100L
;
auto
raw_data
=
DataGen
(
schema
,
num_queries
);
auto
&
info
=
node
->
query_info_
;
info
.
metric_type_
=
"L2"
;
info
.
metric_type_
=
MetricType
::
METRIC_L2
;
info
.
topK_
=
20
;
info
.
field_offset_
=
FieldOffset
(
0
);
node
->
predicate_
=
std
::
nullopt
;
...
...
internal/core/unittest/test_query.cpp
浏览文件 @
7f044fff
...
...
@@ -20,51 +20,12 @@
#include "query/generated/ExecPlanNodeVisitor.h"
#include "query/PlanImpl.h"
#include "segcore/SegmentGrowingImpl.h"
#include "segcore/SegmentSealed.h"
#include "pb/schema.pb.h"
using
namespace
milvus
;
using
namespace
milvus
::
query
;
using
namespace
milvus
::
segcore
;
TEST
(
Query
,
Naive
)
{
SUCCEED
();
using
namespace
milvus
::
wtf
;
std
::
string
dsl_string
=
R"(
{
"bool": {
"must": [
{
"term": {
"A": [
1,
2,
5
]
}
},
{
"range": {
"B": {
"GT": 1,
"LT": 100
}
}
},
{
"vector": {
"Vec": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
]
}
})"
;
}
TEST
(
Query
,
ShowExecutor
)
{
using
namespace
milvus
::
query
;
...
...
@@ -76,7 +37,7 @@ TEST(Query, ShowExecutor) {
int64_t
num_queries
=
100L
;
auto
raw_data
=
DataGen
(
schema
,
num_queries
);
auto
&
info
=
node
->
query_info_
;
info
.
metric_type_
=
"L2"
;
info
.
metric_type_
=
MetricType
::
METRIC_L2
;
info
.
topK_
=
20
;
info
.
field_offset_
=
FieldOffset
(
1000
);
node
->
predicate_
=
std
::
nullopt
;
...
...
@@ -258,6 +219,7 @@ TEST(Query, ExecWithPredicate) {
])"
);
ASSERT_EQ
(
json
.
dump
(
2
),
ref
.
dump
(
2
));
}
TEST
(
Query
,
ExecTerm
)
{
using
namespace
milvus
::
query
;
using
namespace
milvus
::
segcore
;
...
...
internal/core/unittest/test_sealed.cpp
浏览文件 @
7f044fff
...
...
@@ -22,7 +22,7 @@
using
namespace
milvus
;
using
namespace
milvus
::
segcore
;
using
namespace
milvus
;
using
namespace
milvus
::
query
;
TEST
(
Sealed
,
without_predicate
)
{
using
namespace
milvus
::
query
;
...
...
@@ -286,4 +286,41 @@ TEST(Sealed, LoadFieldData) {
ASSERT_EQ
(
chunk_span1
[
i
],
ref1
[
i
]);
ASSERT_EQ
(
chunk_span2
[
i
],
ref2
[
i
]);
}
}
\ No newline at end of file
std
::
string
dsl
=
R"({
"bool": {
"must": [
{
"range": {
"double": {
"GE": -1,
"LT": 1
}
}
},
{
"vector": {
"fakevec": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 5
}
}
}
]
}
})"
;
auto
plan
=
CreatePlan
(
*
schema
,
dsl
);
auto
num_queries
=
5
;
auto
ph_group_raw
=
CreatePlaceholderGroup
(
num_queries
,
16
,
1024
);
auto
ph_group
=
ParsePlaceholderGroup
(
plan
.
get
(),
ph_group_raw
.
SerializeAsString
());
Timestamp
time
=
1000000
;
std
::
vector
<
const
PlaceholderGroup
*>
ph_group_arr
=
{
ph_group
.
get
()};
auto
qr
=
segment
->
Search
(
plan
.
get
(),
ph_group_arr
.
data
(),
&
time
,
1
);
auto
json
=
QueryResultToJson
(
qr
);
std
::
cout
<<
json
.
dump
(
1
);
}
internal/core/unittest/test_utils/DataGen.cpp
0 → 100644
浏览文件 @
7f044fff
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
//
// Created by mike on 1/21/21.
//
internal/distributed/datanode/client.go
浏览文件 @
7f044fff
...
...
@@ -11,15 +11,15 @@ type Client struct {
// GOOSE TODO: add DataNodeClient
}
func
(
c
*
Client
)
Init
()
error
{
func
(
c
*
Client
)
Init
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Start
()
error
{
func
(
c
*
Client
)
Start
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Stop
()
error
{
func
(
c
*
Client
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/distributed/indexservice/service.go
浏览文件 @
7f044fff
...
...
@@ -28,9 +28,8 @@ type Server struct {
loopWg
sync
.
WaitGroup
}
func
(
s
*
Server
)
Init
()
error
{
func
(
s
*
Server
)
Init
()
{
indexservice
.
Params
.
Init
()
return
nil
}
func
(
s
*
Server
)
Start
()
error
{
...
...
@@ -38,9 +37,8 @@ func (s *Server) Start() error {
return
s
.
startIndexServer
()
}
func
(
s
*
Server
)
Stop
()
error
{
func
(
s
*
Server
)
Stop
()
{
s
.
loopWg
.
Wait
()
return
nil
}
func
(
s
*
Server
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
...
...
internal/distributed/querynode/client.go
浏览文件 @
7f044fff
...
...
@@ -13,15 +13,15 @@ type Client struct {
grpcClient
querypb
.
QueryNodeClient
}
func
(
c
*
Client
)
Init
()
error
{
func
(
c
*
Client
)
Init
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Start
()
error
{
func
(
c
*
Client
)
Start
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Stop
()
error
{
func
(
c
*
Client
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/distributed/queryservice/client.go
浏览文件 @
7f044fff
...
...
@@ -9,15 +9,15 @@ type Client struct {
grpcClient
querypb
.
QueryServiceClient
}
func
(
c
*
Client
)
Init
()
error
{
func
(
c
*
Client
)
Init
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Start
()
error
{
func
(
c
*
Client
)
Start
()
{
panic
(
"implement me"
)
}
func
(
c
*
Client
)
Stop
()
error
{
func
(
c
*
Client
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/distributed/queryservice/service.go
浏览文件 @
7f044fff
...
...
@@ -13,15 +13,15 @@ type Server struct {
queryService
queryServiceImpl
.
Interface
}
func
(
s
*
Server
)
Init
()
error
{
func
(
s
*
Server
)
Init
()
{
panic
(
"implement me"
)
}
func
(
s
*
Server
)
Start
()
error
{
func
(
s
*
Server
)
Start
()
{
panic
(
"implement me"
)
}
func
(
s
*
Server
)
Stop
()
error
{
func
(
s
*
Server
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/indexnode/indexnode.go
浏览文件 @
7f044fff
...
...
@@ -45,15 +45,15 @@ type IndexNode struct {
//serviceClient indexservice.Interface // method factory
}
func
(
i
*
IndexNode
)
Init
()
error
{
func
(
i
*
IndexNode
)
Init
()
{
panic
(
"implement me"
)
}
func
(
i
*
IndexNode
)
Start
()
error
{
func
(
i
*
IndexNode
)
Start
()
{
panic
(
"implement me"
)
}
func
(
i
*
IndexNode
)
Stop
()
error
{
func
(
i
*
IndexNode
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/indexservice/indexservice.go
浏览文件 @
7f044fff
...
...
@@ -46,15 +46,15 @@ type IndexService struct {
type
UniqueID
=
typeutil
.
UniqueID
type
Timestamp
=
typeutil
.
Timestamp
func
(
i
*
IndexService
)
Init
()
error
{
func
(
i
*
IndexService
)
Init
()
{
panic
(
"implement me"
)
}
func
(
i
*
IndexService
)
Start
()
error
{
func
(
i
*
IndexService
)
Start
()
{
panic
(
"implement me"
)
}
func
(
i
*
IndexService
)
Stop
()
error
{
func
(
i
*
IndexService
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/proxyservice/proxyservice.go
浏览文件 @
7f044fff
...
...
@@ -13,15 +13,15 @@ type ProxyService struct {
}
func
(
s
ProxyService
)
Init
()
error
{
func
(
s
ProxyService
)
Init
()
{
panic
(
"implement me"
)
}
func
(
s
ProxyService
)
Start
()
error
{
func
(
s
ProxyService
)
Start
()
{
panic
(
"implement me"
)
}
func
(
s
ProxyService
)
Stop
()
error
{
func
(
s
ProxyService
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/querynode/query_node.go
浏览文件 @
7f044fff
...
...
@@ -16,13 +16,10 @@ import (
"context"
"errors"
"fmt"
queryserviceimpl
"github.com/zilliztech/milvus-distributed/internal/queryservice"
"io"
"sync/atomic"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
queryserviceimpl
"github.com/zilliztech/milvus-distributed/internal/queryservice"
"github.com/opentracing/opentracing-go"
"github.com/uber/jaeger-client-go/config"
...
...
@@ -34,7 +31,9 @@ import (
)
type
Node
interface
{
typeutil
.
Service
Init
()
Start
()
Stop
()
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
GetTimeTickChannel
()
(
string
,
error
)
...
...
@@ -124,7 +123,7 @@ func Init() {
Params
.
Init
()
}
func
(
node
*
QueryNode
)
Init
()
error
{
func
(
node
*
QueryNode
)
Init
()
{
registerReq
:=
queryPb
.
RegisterNodeRequest
{
Address
:
&
commonpb
.
Address
{
Ip
:
Params
.
QueryNodeIP
,
...
...
@@ -142,10 +141,9 @@ func (node *QueryNode) Init() error {
// TODO: use response.initParams
Params
.
Init
()
return
nil
}
func
(
node
*
QueryNode
)
Start
()
error
{
func
(
node
*
QueryNode
)
Start
()
{
// todo add connectMaster logic
// init services and manager
node
.
dataSyncService
=
newDataSyncService
(
node
.
queryNodeLoopCtx
,
node
.
replica
)
...
...
@@ -164,10 +162,9 @@ func (node *QueryNode) Start() error {
node
.
stateCode
.
Store
(
internalpb2
.
StateCode_HEALTHY
)
<-
node
.
queryNodeLoopCtx
.
Done
()
return
nil
}
func
(
node
*
QueryNode
)
Stop
()
error
{
func
(
node
*
QueryNode
)
Stop
()
{
node
.
stateCode
.
Store
(
internalpb2
.
StateCode_ABNORMAL
)
node
.
queryNodeLoopCancel
()
...
...
@@ -190,7 +187,6 @@ func (node *QueryNode) Stop() error {
if
node
.
closer
!=
nil
{
node
.
closer
.
Close
()
}
return
nil
}
func
(
node
*
QueryNode
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
...
...
internal/queryservice/queryservice.go
浏览文件 @
7f044fff
...
...
@@ -9,15 +9,15 @@ type QueryService struct {
}
//serverBase interface
func
(
qs
*
QueryService
)
Init
()
error
{
func
(
qs
*
QueryService
)
Init
()
{
panic
(
"implement me"
)
}
func
(
qs
*
QueryService
)
Start
()
error
{
func
(
qs
*
QueryService
)
Start
()
{
panic
(
"implement me"
)
}
func
(
qs
*
QueryService
)
Stop
()
error
{
func
(
qs
*
QueryService
)
Stop
()
{
panic
(
"implement me"
)
}
...
...
internal/util/typeutil/service.go
浏览文件 @
7f044fff
...
...
@@ -5,9 +5,9 @@ import (
)
type
Service
interface
{
Init
()
error
Start
()
error
Stop
()
error
Init
()
Start
()
Stop
()
}
type
Component
interface
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录