Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
a37479d7
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
a37479d7
编写于
4月 08, 2022
作者:
C
Cai Yudong
提交者:
GitHub
4月 08, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Upgrade to knowhere-v1.1.2 to support all index types for mac (#16416)
Signed-off-by:
N
Cai Yudong
<
yudong.cai@zilliz.com
>
上级
9e590a78
变更
16
隐藏空白更改
内联
并排
Showing
16 changed file
with
152 addition
and
184 deletion
+152
-184
.github/workflows/code-checker.yaml
.github/workflows/code-checker.yaml
+1
-1
internal/core/CMakeLists.txt
internal/core/CMakeLists.txt
+1
-20
internal/core/src/index/IndexFactory-inl.h
internal/core/src/index/IndexFactory-inl.h
+2
-1
internal/core/src/index/StringIndexMarisa.cpp
internal/core/src/index/StringIndexMarisa.cpp
+1
-1
internal/core/src/indexbuilder/VecIndexCreator.cpp
internal/core/src/indexbuilder/VecIndexCreator.cpp
+2
-3
internal/core/src/indexbuilder/utils.cpp
internal/core/src/indexbuilder/utils.cpp
+4
-0
internal/core/src/query/SearchBruteForce.cpp
internal/core/src/query/SearchBruteForce.cpp
+0
-25
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
+4
-0
internal/core/src/segcore/FieldIndexing.cpp
internal/core/src/segcore/FieldIndexing.cpp
+0
-6
internal/core/thirdparty/knowhere/CMakeLists.txt
internal/core/thirdparty/knowhere/CMakeLists.txt
+2
-2
internal/core/unittest/CMakeLists.txt
internal/core/unittest/CMakeLists.txt
+34
-42
internal/core/unittest/test_index_c_api.cpp
internal/core/unittest/test_index_c_api.cpp
+3
-0
internal/core/unittest/test_indexing.cpp
internal/core/unittest/test_indexing.cpp
+29
-70
internal/core/unittest/test_query.cpp
internal/core/unittest/test_query.cpp
+39
-0
internal/core/unittest/test_sealed.cpp
internal/core/unittest/test_sealed.cpp
+13
-0
internal/core/unittest/test_utils/indexbuilder_test_utils.h
internal/core/unittest/test_utils/indexbuilder_test_utils.h
+17
-13
未找到文件。
.github/workflows/code-checker.yaml
浏览文件 @
a37479d7
...
...
@@ -109,7 +109,7 @@ jobs:
if [[ ! -d "/var/tmp/ccache" ]];then
mkdir -p /var/tmp/ccache
fi
brew install boost libomp ninja tbb ccache
brew install boost libomp ninja tbb
openblas
ccache
make check-proto-product && make verifiers
centos
:
name
:
Code Checker CentOS
7
...
...
internal/core/CMakeLists.txt
浏览文件 @
a37479d7
...
...
@@ -31,26 +31,7 @@ project(core)
include
(
CheckCXXCompilerFlag
)
if
(
APPLE
)
message
(
STATUS
"==============Darwin Environment=============="
)
check_cxx_compiler_flag
(
-std=c++11 HAS_STD_CPP11_FLAG
)
if
(
HAS_STD_CPP11_FLAG
)
add_compile_options
(
-std=c++11
)
endif
()
if
(
CMAKE_C_COMPILER_ID MATCHES
"Clang"
)
set
(
OpenMP_C
"
${
CMAKE_C_COMPILER
}
"
CACHE STRING
""
FORCE
)
set
(
OpenMP_C_FLAGS
"-fopenmp=libomp -Wno-unused-command-line-argument"
CACHE STRING
""
FORCE
)
set
(
OpenMP_C_LIB_NAMES
"libomp"
"libgomp"
"libiomp5"
CACHE STRING
""
FORCE
)
set
(
OpenMP_libomp_LIBRARY
${
OpenMP_C_LIB_NAMES
}
CACHE STRING
""
FORCE
)
set
(
OpenMP_libgomp_LIBRARY
${
OpenMP_C_LIB_NAMES
}
CACHE STRING
""
FORCE
)
set
(
OpenMP_libiomp5_LIBRARY
${
OpenMP_C_LIB_NAMES
}
CACHE STRING
""
FORCE
)
endif
()
if
(
CMAKE_CXX_COMPILER_ID MATCHES
"Clang"
)
set
(
OpenMP_CXX
"
${
CMAKE_CXX_COMPILER
}
"
CACHE STRING
""
FORCE
)
set
(
OpenMP_CXX_FLAGS
"-fopenmp=libomp -Wno-unused-command-line-argument"
CACHE STRING
""
FORCE
)
set
(
OpenMP_CXX_LIB_NAMES
"libomp"
"libgomp"
"libiomp5"
CACHE STRING
""
FORCE
)
set
(
OpenMP_libomp_LIBRARY
${
OpenMP_CXX_LIB_NAMES
}
CACHE STRING
""
FORCE
)
set
(
OpenMP_libgomp_LIBRARY
${
OpenMP_CXX_LIB_NAMES
}
CACHE STRING
""
FORCE
)
set
(
OpenMP_libiomp5_LIBRARY
${
OpenMP_CXX_LIB_NAMES
}
CACHE STRING
""
FORCE
)
endif
()
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib"
)
elseif
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Linux"
)
message
(
STATUS
"==============Linux Environment==============="
)
set
(
LINUX TRUE
)
...
...
internal/core/src/index/IndexFactory-inl.h
浏览文件 @
a37479d7
...
...
@@ -34,8 +34,9 @@ inline ScalarIndexPtr<std::string>
IndexFactory
::
CreateIndex
(
const
std
::
string
&
index_type
)
{
#ifdef __linux__
return
CreateStringIndexMarisa
();
#e
ndif
#e
lse
throw
std
::
runtime_error
(
"unsupported platform"
);
#endif
}
}
// namespace milvus::scalar
internal/core/src/index/StringIndexMarisa.cpp
浏览文件 @
a37479d7
...
...
@@ -80,7 +80,7 @@ StringIndexMarisa::Serialize(const Config& config) {
res_set
.
Append
(
MARISA_TRIE_INDEX
,
index_data
,
size
);
res_set
.
Append
(
MARISA_STR_IDS
,
str_ids
,
str_ids_len
);
knowhere
::
Disassemble
(
4
*
1024
*
1024
,
res_set
);
knowhere
::
Disassemble
(
res_set
,
config
);
return
res_set
;
}
...
...
internal/core/src/indexbuilder/VecIndexCreator.cpp
浏览文件 @
a37479d7
...
...
@@ -238,10 +238,9 @@ VecIndexCreator::Serialize() {
std
::
shared_ptr
<
uint8_t
[]
>
raw_data
(
new
uint8_t
[
raw_data_
.
size
()],
std
::
default_delete
<
uint8_t
[]
>
());
memcpy
(
raw_data
.
get
(),
raw_data_
.
data
(),
raw_data_
.
size
());
ret
.
Append
(
RAW_DATA
,
raw_data
,
raw_data_
.
size
());
auto
slice_size
=
get_index_file_slice_size
();
// https://github.com/milvus-io/milvus/issues/6421
// Disassemble will only divide the raw vectors, other keys were already divided
knowhere
::
Disassemble
(
slice_size
*
1024
*
1024
,
ret
);
knowhere
::
Disassemble
(
ret
,
config_
);
}
return
ret
;
}
...
...
@@ -300,7 +299,7 @@ VecIndexCreator::get_index_file_slice_size() {
if
(
config_
.
contains
(
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
))
{
return
config_
[
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
].
get
<
int64_t
>
();
}
return
4
;
// by default
return
knowhere
::
index_file_slice_size
;
// by default
}
std
::
unique_ptr
<
VecIndexCreator
::
QueryResult
>
...
...
internal/core/src/indexbuilder/utils.cpp
浏览文件 @
a37479d7
...
...
@@ -25,7 +25,9 @@ std::vector<std::string>
NM_List
()
{
static
std
::
vector
<
std
::
string
>
ret
{
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
,
#ifdef MILVUS_SUPPORT_NSG
knowhere
::
IndexEnum
::
INDEX_NSG
,
#endif
knowhere
::
IndexEnum
::
INDEX_RHNSWFlat
,
};
return
ret
;
...
...
@@ -53,7 +55,9 @@ Need_ID_List() {
std
::
vector
<
std
::
string
>
Need_BuildAll_list
()
{
static
std
::
vector
<
std
::
string
>
ret
{
#ifdef MILVUS_SUPPORT_NSG
knowhere
::
IndexEnum
::
INDEX_NSG
,
#endif
};
return
ret
;
}
...
...
internal/core/src/query/SearchBruteForce.cpp
浏览文件 @
a37479d7
...
...
@@ -12,15 +12,8 @@
#include <string>
#include <vector>
#ifdef __APPLE__
#include "knowhere/index/vector_index/impl/bruteforce/BruteForce.h"
#include "knowhere/common/Heap.h"
#elif defined(__linux__) || defined(__MINGW64__)
#include <faiss/utils/BinaryDistance.h>
#include <faiss/utils/distances.h>
#else
#error "Unsupported OS environment.";
#endif
#include "SearchBruteForce.h"
#include "SubSearchResult.h"
...
...
@@ -42,7 +35,6 @@ raw_search(MetricType metric_type,
float
*
D
,
idx_t
*
labels
,
const
BitsetView
bitset
)
{
#if defined(__linux__) || defined(__MINGW64__)
using
namespace
faiss
;
// NOLINT
if
(
metric_type
==
METRIC_Jaccard
||
metric_type
==
METRIC_Tanimoto
)
{
float_maxheap_array_t
res
=
{
size_t
(
n
),
size_t
(
k
),
labels
,
D
};
...
...
@@ -70,9 +62,6 @@ raw_search(MetricType metric_type,
std
::
string
(
"binary search not support metric type: "
)
+
segcore
::
MetricTypeToString
(
metric_type
);
PanicInfo
(
msg
);
}
#else
PanicInfo
(
"Unsupported brute force for binary search on current OS environment!"
);
#endif
}
SubSearchResult
...
...
@@ -111,17 +100,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
SubSearchResult
sub_qr
(
num_queries
,
topk
,
metric_type
,
round_decimal
);
auto
query_data
=
reinterpret_cast
<
const
float
*>
(
dataset
.
query_data
);
auto
chunk_data
=
reinterpret_cast
<
const
float
*>
(
chunk_data_raw
);
#ifdef __APPLE__
if
(
metric_type
==
MetricType
::
METRIC_L2
)
{
knowhere
::
float_maxheap_array_t
buf
{(
size_t
)
num_queries
,
(
size_t
)
topk
,
sub_qr
.
get_ids
(),
sub_qr
.
get_distances
()};
knowhere
::
knn_L2sqr_sse
(
query_data
,
chunk_data
,
dim
,
num_queries
,
size_per_chunk
,
&
buf
,
bitset
);
}
else
{
knowhere
::
float_minheap_array_t
buf
{(
size_t
)
num_queries
,
(
size_t
)
topk
,
sub_qr
.
get_ids
(),
sub_qr
.
get_distances
()};
knowhere
::
knn_inner_product_sse
(
query_data
,
chunk_data
,
dim
,
num_queries
,
size_per_chunk
,
&
buf
,
bitset
);
}
#elif defined(__linux__) || defined(__MINGW64__)
if
(
metric_type
==
MetricType
::
METRIC_L2
)
{
faiss
::
float_maxheap_array_t
buf
{(
size_t
)
num_queries
,
(
size_t
)
topk
,
sub_qr
.
get_ids
(),
sub_qr
.
get_distances
()};
faiss
::
knn_L2sqr
(
query_data
,
chunk_data
,
dim
,
num_queries
,
size_per_chunk
,
&
buf
,
nullptr
,
bitset
);
...
...
@@ -129,9 +107,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
faiss
::
float_minheap_array_t
buf
{(
size_t
)
num_queries
,
(
size_t
)
topk
,
sub_qr
.
get_ids
(),
sub_qr
.
get_distances
()};
faiss
::
knn_inner_product
(
query_data
,
chunk_data
,
dim
,
num_queries
,
size_per_chunk
,
&
buf
,
bitset
);
}
#else
#error "Unsupported OS environment!";
#endif
sub_qr
.
round_values
();
return
sub_qr
;
}
...
...
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
浏览文件 @
a37479d7
...
...
@@ -41,11 +41,15 @@ InferIndexType(const Json& search_params) {
namespace
ip
=
knowhere
::
IndexParams
;
namespace
ie
=
knowhere
::
IndexEnum
;
list
.
emplace
(
ip
::
nprobe
,
ie
::
INDEX_FAISS_IVFFLAT
);
#ifdef MILVUS_SUPPORT_NSG
list
.
emplace
(
ip
::
search_length
,
ie
::
INDEX_NSG
);
#endif
list
.
emplace
(
ip
::
ef
,
ie
::
INDEX_HNSW
);
list
.
emplace
(
ip
::
search_k
,
ie
::
INDEX_ANNOY
);
#ifdef MILVUS_SUPPORT_NGT
list
.
emplace
(
ip
::
max_search_edges
,
ie
::
INDEX_NGTONNG
);
list
.
emplace
(
ip
::
epsilon
,
ie
::
INDEX_NGTONNG
);
#endif
return
list
;
}();
auto
dbg_str
=
search_params
.
dump
();
...
...
internal/core/src/segcore/FieldIndexing.cpp
浏览文件 @
a37479d7
...
...
@@ -14,9 +14,7 @@
#include <index/ScalarIndexSort.h>
#include "common/SystemProperty.h"
#if defined(__linux__) || defined(__MINGW64__)
#include "knowhere/index/vector_index/IndexIVF.h"
#endif
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "segcore/FieldIndexing.h"
...
...
@@ -24,7 +22,6 @@ namespace milvus::segcore {
void
VectorFieldIndexing
::
BuildIndexRange
(
int64_t
ack_beg
,
int64_t
ack_end
,
const
VectorBase
*
vec_base
)
{
#if defined(__linux__) || defined(__MINGW64__)
AssertInfo
(
field_meta_
.
get_data_type
()
==
DataType
::
VECTOR_FLOAT
,
"Data type of vector field is not VECTOR_FLOAT"
);
auto
dim
=
field_meta_
.
get_dim
();
...
...
@@ -43,9 +40,6 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec
indexing
->
AddWithoutIds
(
dataset
,
conf
);
data_
[
chunk_id
]
=
std
::
move
(
indexing
);
}
#else
throw
std
::
runtime_error
(
"Unsupported BuildIndexRange on current platform!"
);
#endif
}
knowhere
::
Config
...
...
internal/core/thirdparty/knowhere/CMakeLists.txt
浏览文件 @
a37479d7
...
...
@@ -11,8 +11,8 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set
(
KNOWHERE_VERSION v1.1.
1
)
set
(
KNOWHERE_SOURCE_MD5
"
3f6dff91dc947a9b003434b8901e9130
"
)
set
(
KNOWHERE_VERSION v1.1.
2
)
set
(
KNOWHERE_SOURCE_MD5
"
7c6043bfaa073c6580a1c5b1e904c995
"
)
if
(
DEFINED ENV{MILVUS_KNOWHERE_URL}
)
set
(
KNOWHERE_SOURCE_URL
"$ENV{MILVUS_KNOWHERE_URL}"
)
...
...
internal/core/unittest/CMakeLists.txt
浏览文件 @
a37479d7
...
...
@@ -14,37 +14,45 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty)
add_definitions
(
-DMILVUS_TEST_SEGCORE_YAML_PATH=
"
${
CMAKE_SOURCE_DIR
}
/unittest/test_utils/test_segcore.yaml"
)
# TODO: better to use ls/find pattern
set
(
MILVUS_TEST_FILES
init_gtest.cpp
test_binary.cpp
test_bitmap.cpp
test_bool_index.cpp
test_common.cpp
test_concurrent_vector.cpp
test_conf_adapter_mgr.cpp
test_c_api.cpp
test_expr.cpp
test_indexing.cpp
test_index_c_api.cpp
test_index_wrapper.cpp
test_init.cpp
test_plan_proto.cpp
test_query.cpp
test_reduce.cpp
test_reduce_c.cpp
test_retrieve.cpp
test_scalar_index.cpp
test_sealed.cpp
test_segcore.cpp
test_similarity_corelation.cpp
test_span.cpp
test_timestamp_index.cpp
test_utils.cpp
)
# These 2 cases cannot run in mac because of tbb build error
if
(
LINUX
)
# TODO: better to use ls/find pattern
set
(
MILVUS_TEST_FILES
init_gtest.cpp
test_binary.cpp
test_bitmap.cpp
test_common.cpp
test_concurrent_vector.cpp
test_c_api.cpp
test_expr.cpp
test_retrieve.cpp
test_indexing.cpp
test_index_wrapper.cpp
test_init.cpp
test_plan_proto.cpp
test_query.cpp
test_reduce.cpp
test_sealed.cpp
test_segcore.cpp
test_span.cpp
test_timestamp_index.cpp
test_reduce_c.cpp
test_conf_adapter_mgr.cpp
test_similarity_corelation.cpp
test_utils.cpp
${
MILVUS_TEST_FILES
}
test_scalar_index_creator.cpp
test_index_c_api.cpp
test_scalar_index.cpp
test_string_index.cpp
test_bool_index.cpp
)
endif
()
if
(
LINUX
)
# check if memory leak exists in index builder
set
(
INDEX_BUILDER_TEST_FILES
test_index_wrapper.cpp
...
...
@@ -68,22 +76,6 @@ if (LINUX)
install
(
TARGETS index_builder_test DESTINATION unittest
)
endif
()
if
(
APPLE
)
set
(
MILVUS_TEST_FILES
init_gtest.cpp
test_common.cpp
test_concurrent_vector.cpp
test_conf_adapter_mgr.cpp
test_plan_proto.cpp
test_reduce.cpp
test_reduce_c.cpp
test_segcore.cpp
test_timestamp_index.cpp
test_similarity_corelation.cpp
test_utils.cpp
)
endif
()
add_executable
(
all_tests
${
MILVUS_TEST_FILES
}
)
...
...
internal/core/unittest/test_index_c_api.cpp
浏览文件 @
a37479d7
...
...
@@ -236,6 +236,8 @@ TEST(CInt64IndexTest, All) {
}
}
// disable this case since marisa not supported in mac
#ifdef __linux__
TEST
(
CStringIndexTest
,
All
)
{
auto
strs
=
GenStrArr
(
NB
);
schemapb
::
StringArray
str_arr
;
...
...
@@ -288,3 +290,4 @@ TEST(CStringIndexTest, All) {
delete
[](
char
*
)
str_ds
->
Get
<
const
void
*>
(
knowhere
::
meta
::
TENSOR
);
}
#endif
internal/core/unittest/test_indexing.cpp
浏览文件 @
a37479d7
...
...
@@ -298,82 +298,41 @@ TEST(Indexing, BinaryBruteForce) {
auto
json
=
SearchResultToJson
(
sr
);
std
::
cout
<<
json
.
dump
(
2
);
#ifdef __linux__
auto
ref
=
json
::
parse
(
R"(
[
[
[
"1024->0.000000",
"48942->0.642000",
"18494->0.644000",
"68225->0.644000",
"93557->0.644000"
],
[
"1025->0.000000",
"73557->0.641000",
"53086->0.643000",
"9737->0.643000",
"62855->0.644000"
],
[
"1026->0.000000",
"62904->0.644000",
"46758->0.644000",
"57969->0.645000",
"98113->0.646000"
],
[
"1027->0.000000",
"92446->0.638000",
"96034->0.640000",
"92129->0.644000",
"45887->0.644000"
],
[
"1028->0.000000",
"22992->0.643000",
"73903->0.644000",
"19969->0.645000",
"65178->0.645000"
],
[
"1029->0.000000",
"19776->0.641000",
"15166->0.642000",
"85470->0.642000",
"16730->0.643000"
],
[
"1030->0.000000",
"55939->0.640000",
"84253->0.643000",
"31958->0.644000",
"11667->0.646000"
],
[
"1031->0.000000",
"89536->0.637000",
"61622->0.638000",
"9275->0.639000",
"91403->0.640000"
],
[
"1032->0.000000",
"69504->0.642000",
"23414->0.644000",
"48770->0.645000",
"23231->0.645000"
],
[
"1033->0.000000",
"33540->0.636000",
"25310->0.640000",
"18576->0.640000",
"73729->0.642000"
]
[ "1024->0.000000", "48942->0.642000", "18494->0.644000", "68225->0.644000", "93557->0.644000" ],
[ "1025->0.000000", "73557->0.641000", "53086->0.643000", "9737->0.643000", "62855->0.644000" ],
[ "1026->0.000000", "62904->0.644000", "46758->0.644000", "57969->0.645000", "98113->0.646000" ],
[ "1027->0.000000", "92446->0.638000", "96034->0.640000", "92129->0.644000", "45887->0.644000" ],
[ "1028->0.000000", "22992->0.643000", "73903->0.644000", "19969->0.645000", "65178->0.645000" ],
[ "1029->0.000000", "19776->0.641000", "15166->0.642000", "85470->0.642000", "16730->0.643000" ],
[ "1030->0.000000", "55939->0.640000", "84253->0.643000", "31958->0.644000", "11667->0.646000" ],
[ "1031->0.000000", "89536->0.637000", "61622->0.638000", "9275->0.639000", "91403->0.640000" ],
[ "1032->0.000000", "69504->0.642000", "23414->0.644000", "48770->0.645000", "23231->0.645000" ],
[ "1033->0.000000", "33540->0.636000", "25310->0.640000", "18576->0.640000", "73729->0.642000" ]
]
]
)"
);
#else // for mac
auto
ref
=
json
::
parse
(
R"(
[
[
[ "1024->0.000000", "59169->0.645000", "98548->0.646000", "3356->0.646000", "90373->0.647000" ],
[ "1025->0.000000", "61245->0.638000", "95271->0.639000", "31087->0.639000", "31549->0.640000" ],
[ "1026->0.000000", "65225->0.648000", "35750->0.648000", "14971->0.649000", "75385->0.649000" ],
[ "1027->0.000000", "70158->0.640000", "27076->0.640000", "3407->0.641000", "59527->0.641000" ],
[ "1028->0.000000", "45757->0.645000", "3356->0.645000", "77230->0.646000", "28690->0.647000" ],
[ "1029->0.000000", "13291->0.642000", "24960->0.643000", "83770->0.643000", "88244->0.643000" ],
[ "1030->0.000000", "96807->0.641000", "39920->0.643000", "62943->0.644000", "12603->0.644000" ],
[ "1031->0.000000", "65769->0.648000", "60493->0.648000", "48738->0.648000", "4353->0.648000" ],
[ "1032->0.000000", "57827->0.637000", "8213->0.638000", "22221->0.639000", "23328->0.640000" ],
[ "1033->0.000000", "676->0.645000", "91430->0.646000", "85353->0.646000", "6014->0.646000" ]
]
]
)"
);
#endif
auto
json_str
=
json
.
dump
(
2
);
auto
ref_str
=
ref
.
dump
(
2
);
ASSERT_EQ
(
json_str
,
ref_str
);
...
...
internal/core/unittest/test_query.cpp
浏览文件 @
a37479d7
...
...
@@ -186,6 +186,7 @@ TEST(Query, ExecWithPredicateLoader) {
int
topk
=
5
;
Json
json
=
SearchResultToJson
(
*
sr
);
#ifdef __linux__
auto
ref
=
json
::
parse
(
R"(
[
[
...
...
@@ -196,6 +197,18 @@ TEST(Query, ExecWithPredicateLoader) {
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])"
);
#else // for mac
auto
ref
=
json
::
parse
(
R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"],
["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"],
["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"]
]
])"
);
#endif
std
::
cout
<<
json
.
dump
(
2
);
ASSERT_EQ
(
json
.
dump
(
2
),
ref
.
dump
(
2
));
}
...
...
@@ -301,6 +314,7 @@ TEST(Query, ExecWithPredicate) {
int
topk
=
5
;
Json
json
=
SearchResultToJson
(
*
sr
);
#ifdef __linux__
auto
ref
=
json
::
parse
(
R"(
[
[
...
...
@@ -311,6 +325,18 @@ TEST(Query, ExecWithPredicate) {
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])"
);
#else // for mac
auto
ref
=
json
::
parse
(
R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"],
["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"],
["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"]
]
])"
);
#endif
std
::
cout
<<
json
.
dump
(
2
);
ASSERT_EQ
(
json
.
dump
(
2
),
ref
.
dump
(
2
));
}
...
...
@@ -498,6 +524,7 @@ TEST(Query, ExecWithoutPredicate) {
std
::
vector
<
std
::
vector
<
std
::
string
>>
results
;
int
topk
=
5
;
auto
json
=
SearchResultToJson
(
*
sr
);
#ifdef __linux__
auto
ref
=
json
::
parse
(
R"(
[
[
...
...
@@ -508,6 +535,18 @@ TEST(Query, ExecWithoutPredicate) {
["66353->5.696000", "41087->5.917000", "24554->6.195000", "68019->6.654000", "97780->6.811000"]
]
])"
);
#else // for mac
auto
ref
=
json
::
parse
(
R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "78227->4.808000", "71547->5.125000"],
["96984->4.192000", "45733->4.912000", "32891->5.016000", "65514->6.011000", "89328->6.138000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "76784->5.195000"],
["99625->6.129000", "86582->6.900000", "60608->7.285000", "10069->7.388000", "89982->7.672000"],
["37759->3.581000", "45814->4.872000", "97019->5.557000", "23626->5.839000", "53543->5.844000"]
]
])"
);
#endif
std
::
cout
<<
json
.
dump
(
2
);
ASSERT_EQ
(
json
.
dump
(
2
),
ref
.
dump
(
2
));
}
...
...
internal/core/unittest/test_sealed.cpp
浏览文件 @
a37479d7
...
...
@@ -298,6 +298,7 @@ TEST(Sealed, LoadFieldData) {
ASSERT_EQ
(
json
.
dump
(
-
2
),
json2
.
dump
(
-
2
));
segment
->
DropFieldData
(
double_id
);
ASSERT_ANY_THROW
(
segment
->
Search
(
plan
.
get
(),
*
ph_group
,
time
));
#ifdef __linux__
auto
std_json
=
Json
::
parse
(
R"(
[
[
...
...
@@ -308,6 +309,18 @@ TEST(Sealed, LoadFieldData) {
["66353->5.696000", "30664->5.881000", "41087->5.917000", "10393->6.633000", "90215->7.202000"]
]
])"
);
#else // for mac
auto
std_json
=
Json
::
parse
(
R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "86706->5.991000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "82365->4.725000", "74834->5.009000", "79995->5.725000", "33359->5.816000"],
["99625->6.129000", "86582->6.900000", "85934->7.792000", "60450->8.087000", "19257->8.530000"],
["37759->3.581000", "31292->5.780000", "98124->6.216000", "63535->6.439000", "11707->6.553000"]
]
])"
);
#endif
ASSERT_EQ
(
std_json
.
dump
(
-
2
),
json
.
dump
(
-
2
));
}
...
...
internal/core/unittest/test_utils/indexbuilder_test_utils.h
浏览文件 @
a37479d7
...
...
@@ -56,7 +56,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
meta
::
DIM
,
DIM
},
{
knowhere
::
meta
::
TOPK
,
K
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFPQ
)
{
return
knowhere
::
Config
{
...
...
@@ -67,7 +67,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
m
,
4
},
{
knowhere
::
IndexParams
::
nbits
,
8
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_FAISS_IVFFLAT
)
{
return
knowhere
::
Config
{
...
...
@@ -76,7 +76,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
nlist
,
16
},
{
knowhere
::
IndexParams
::
nprobe
,
4
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
#ifdef MILVUS_GPU_VERSION
{
knowhere
::
meta
::
DEVICEID
,
DEVICEID
},
#endif
...
...
@@ -89,7 +89,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
nprobe
,
4
},
{
knowhere
::
IndexParams
::
nbits
,
8
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
#ifdef MILVUS_GPU_VERSION
{
knowhere
::
meta
::
DEVICEID
,
DEVICEID
},
#endif
...
...
@@ -103,7 +103,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
m
,
4
},
{
knowhere
::
IndexParams
::
nbits
,
8
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_FAISS_BIN_IDMAP
)
{
return
knowhere
::
Config
{
...
...
@@ -111,6 +111,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
meta
::
TOPK
,
K
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
};
#ifdef MILVUS_SUPPORT_NSG
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_NSG
)
{
return
knowhere
::
Config
{
{
knowhere
::
meta
::
DIM
,
DIM
},
...
...
@@ -123,20 +124,21 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
candidate
,
100
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
};
#endif
#ifdef MILVUS_SUPPORT_SPTAG
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_SPTAG_KDT_RNT
)
{
return
knowhere
::
Config
{
{
knowhere
::
meta
::
DIM
,
DIM
},
// {knowhere::meta::TOPK, 10},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_SPTAG_BKT_RNT
)
{
return
knowhere
::
Config
{
{
knowhere
::
meta
::
DIM
,
DIM
},
// {knowhere::meta::TOPK, 10},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
#endif
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_HNSW
)
{
...
...
@@ -152,7 +154,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
n_trees
,
4
},
{
knowhere
::
IndexParams
::
search_k
,
100
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_RHNSWFlat
)
{
return
knowhere
::
Config
{
...
...
@@ -162,7 +164,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
efConstruction
,
200
},
{
knowhere
::
IndexParams
::
ef
,
200
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_RHNSWPQ
)
{
return
knowhere
::
Config
{
...
...
@@ -172,7 +174,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
efConstruction
,
200
},
{
knowhere
::
IndexParams
::
ef
,
200
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
{
knowhere
::
IndexParams
::
PQM
,
8
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_RHNSWSQ
)
{
...
...
@@ -183,8 +185,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
efConstruction
,
200
},
{
knowhere
::
IndexParams
::
ef
,
200
},
{
knowhere
::
Metric
::
TYPE
,
metric_type
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
#ifdef MILVUS_SUPPORT_NGT
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_NGTPANNG
)
{
return
knowhere
::
Config
{
{
knowhere
::
meta
::
DIM
,
DIM
},
...
...
@@ -195,7 +198,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
max_search_edges
,
50
},
{
knowhere
::
IndexParams
::
forcedly_pruned_edge_size
,
60
},
{
knowhere
::
IndexParams
::
selectively_pruned_edge_size
,
30
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
}
else
if
(
index_type
==
knowhere
::
IndexEnum
::
INDEX_NGTONNG
)
{
return
knowhere
::
Config
{
...
...
@@ -207,8 +210,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{
knowhere
::
IndexParams
::
max_search_edges
,
50
},
{
knowhere
::
IndexParams
::
outgoing_edge_size
,
5
},
{
knowhere
::
IndexParams
::
incoming_edge_size
,
40
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
4
},
{
knowhere
::
INDEX_FILE_SLICE_SIZE_IN_MEGABYTE
,
knowhere
::
index_file_slice_size
},
};
#endif
}
return
knowhere
::
Config
();
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录