Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
9d95e472
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9d95e472
编写于
10月 31, 2019
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'main/0.5.1' into 0.5.1
Former-commit-id: 980bc7ea1b399e9bcd494195c41ae7e775c45b8c
上级
47bbedba
a6f08ac9
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
304 addition
and
253 deletion
+304
-253
CHANGELOG.md
CHANGELOG.md
+2
-0
core/CMakeLists.txt
core/CMakeLists.txt
+5
-1
core/cmake/ThirdPartyPackages.cmake
core/cmake/ThirdPartyPackages.cmake
+193
-182
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+1
-1
core/src/index/cmake/ThirdPartyPackagesCore.cmake
core/src/index/cmake/ThirdPartyPackagesCore.cmake
+2
-1
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
...c/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
+88
-58
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
+12
-10
core/src/scheduler/optimizer/LargeSQ8HPass.h
core/src/scheduler/optimizer/LargeSQ8HPass.h
+1
-0
未找到文件。
CHANGELOG.md
浏览文件 @
9d95e472
...
...
@@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.5.1 (TODO)
## Bug
-
\#
134 - JFrog cache error
## Feature
-
\#
90 - The server start error messages could be improved to enhance user experience
...
...
@@ -21,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
118 - Using shared_ptr instead of weak_ptr to avoid performance loss
-
\#
122 - Add unique id for Job
-
\#
130 - Set task state MOVED after resource copy it completed
-
\#
149 - Improve large query optimizer pass
## Task
...
...
core/CMakeLists.txt
浏览文件 @
9d95e472
...
...
@@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message
(
STATUS
"Build time =
${
BUILD_TIME
}
"
)
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
execute_process
(
COMMAND
"git"
symbolic-ref --short HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
execute_process
(
COMMAND
"git"
rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
if
(
GIT_BRANCH_NAME STREQUAL
""
)
execute_process
(
COMMAND
"git"
symbolic-ref --short -q HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
endif
()
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
GET_GIT_BRANCH_NAME
(
GIT_BRANCH_NAME
)
message
(
STATUS
"GIT_BRANCH_NAME =
${
GIT_BRANCH_NAME
}
"
)
if
(
NOT GIT_BRANCH_NAME STREQUAL
""
)
string
(
REGEX REPLACE
"
\n
"
""
GIT_BRANCH_NAME
${
GIT_BRANCH_NAME
}
)
endif
()
...
...
core/cmake/ThirdPartyPackages.cmake
浏览文件 @
9d95e472
...
...
@@ -30,11 +30,11 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
message
(
STATUS
"Using
${
MILVUS_DEPENDENCY_SOURCE
}
approach to find dependencies"
)
# For each dependency, set dependency source to global default, if unset
foreach
(
DEPENDENCY
${
MILVUS_THIRDPARTY_DEPENDENCIES
}
)
if
(
"
${${
DEPENDENCY
}
_SOURCE
}
"
STREQUAL
""
)
foreach
(
DEPENDENCY
${
MILVUS_THIRDPARTY_DEPENDENCIES
}
)
if
(
"
${${
DEPENDENCY
}
_SOURCE
}
"
STREQUAL
""
)
set
(
${
DEPENDENCY
}
_SOURCE
${
MILVUS_DEPENDENCY_SOURCE
}
)
endif
()
endforeach
()
endif
()
endforeach
()
macro
(
build_dependency DEPENDENCY_NAME
)
if
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"GTest"
)
...
...
@@ -47,17 +47,17 @@ macro(build_dependency DEPENDENCY_NAME)
build_sqlite
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"SQLite_ORM"
)
build_sqlite_orm
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"yaml-cpp"
)
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"yaml-cpp"
)
build_yamlcpp
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"libunwind"
)
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"libunwind"
)
build_libunwind
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"gperftools"
)
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"gperftools"
)
build_gperftools
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"GRPC"
)
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"GRPC"
)
build_grpc
()
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"ZLIB"
)
elseif
(
"
${
DEPENDENCY_NAME
}
"
STREQUAL
"ZLIB"
)
build_zlib
()
else
()
else
()
message
(
FATAL_ERROR
"Unknown thirdparty dependency to build:
${
DEPENDENCY_NAME
}
"
)
endif
()
endmacro
()
...
...
@@ -66,28 +66,28 @@ endmacro()
# Identify OS
if
(
UNIX
)
if
(
APPLE
)
set
(
CMAKE_OS_NAME
"osx"
CACHE STRING
"Operating system name"
FORCE
)
set
(
CMAKE_OS_NAME
"osx"
CACHE STRING
"Operating system name"
FORCE
)
else
(
APPLE
)
## Check for Debian GNU/Linux ________________
find_file
(
DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
)
find_file
(
DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
)
if
(
DEBIAN_FOUND
)
set
(
CMAKE_OS_NAME
"debian"
CACHE STRING
"Operating system name"
FORCE
)
set
(
CMAKE_OS_NAME
"debian"
CACHE STRING
"Operating system name"
FORCE
)
endif
(
DEBIAN_FOUND
)
## Check for Fedora _________________________
find_file
(
FEDORA_FOUND fedora-release
PATHS /etc
)
find_file
(
FEDORA_FOUND fedora-release
PATHS /etc
)
if
(
FEDORA_FOUND
)
set
(
CMAKE_OS_NAME
"fedora"
CACHE STRING
"Operating system name"
FORCE
)
set
(
CMAKE_OS_NAME
"fedora"
CACHE STRING
"Operating system name"
FORCE
)
endif
(
FEDORA_FOUND
)
## Check for RedHat _________________________
find_file
(
REDHAT_FOUND redhat-release inittab.RH
PATHS /etc
)
find_file
(
REDHAT_FOUND redhat-release inittab.RH
PATHS /etc
)
if
(
REDHAT_FOUND
)
set
(
CMAKE_OS_NAME
"redhat"
CACHE STRING
"Operating system name"
FORCE
)
set
(
CMAKE_OS_NAME
"redhat"
CACHE STRING
"Operating system name"
FORCE
)
endif
(
REDHAT_FOUND
)
## Extra check for Ubuntu ____________________
if
(
DEBIAN_FOUND
)
...
...
@@ -96,18 +96,25 @@ if (UNIX)
## a first superficial inspection a system will
## be considered as Debian, which signifies an
## extra check is required.
find_file
(
UBUNTU_EXTRA legal issue
PATHS /etc
)
find_file
(
UBUNTU_EXTRA legal issue
PATHS /etc
)
if
(
UBUNTU_EXTRA
)
## Scan contents of file
file
(
STRINGS
${
UBUNTU_EXTRA
}
UBUNTU_FOUND
REGEX Ubuntu
)
file
(
STRINGS
${
UBUNTU_EXTRA
}
UBUNTU_FOUND
REGEX Ubuntu
)
## Check result of string search
if
(
UBUNTU_FOUND
)
set
(
CMAKE_OS_NAME
"ubuntu"
CACHE STRING
"Operating system name"
FORCE
)
set
(
DEBIAN_FOUND FALSE
)
set
(
CMAKE_OS_NAME
"ubuntu"
CACHE STRING
"Operating system name"
FORCE
)
set
(
DEBIAN_FOUND FALSE
)
find_program
(
LSB_RELEASE_EXEC lsb_release
)
execute_process
(
COMMAND
${
LSB_RELEASE_EXEC
}
-rs
OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
STRING
(
REGEX REPLACE
"
\\
."
"_"
UBUNTU_VERSION
"
${
LSB_RELEASE_ID_SHORT
}
"
)
endif
(
UBUNTU_FOUND
)
endif
(
UBUNTU_EXTRA
)
endif
(
DEBIAN_FOUND
)
...
...
@@ -120,36 +127,40 @@ set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# JFrog
if
(
NOT DEFINED USE_JFROG_CACHE
)
if
(
NOT DEFINED USE_JFROG_CACHE
)
set
(
USE_JFROG_CACHE
"OFF"
)
endif
()
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
DEFINED ENV{JFROG_ARTFACTORY_URL}
)
endif
()
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
DEFINED ENV{JFROG_ARTFACTORY_URL}
)
set
(
JFROG_ARTFACTORY_URL
"$ENV{JFROG_ARTFACTORY_URL}"
)
endif
()
if
(
NOT DEFINED JFROG_ARTFACTORY_URL
)
endif
()
if
(
NOT DEFINED JFROG_ARTFACTORY_URL
)
message
(
FATAL_ERROR
"JFROG_ARTFACTORY_URL is not set"
)
endif
()
set
(
JFROG_ARTFACTORY_CACHE_URL
"
${
JFROG_ARTFACTORY_URL
}
/milvus/thirdparty/cache/
${
CMAKE_OS_NAME
}
/
${
MILVUS_BUILD_ARCH
}
/
${
BUILD_TYPE
}
"
)
if
(
DEFINED ENV{JFROG_USER_NAME}
)
endif
()
if
(
UBUNTU_FOUND
)
set
(
JFROG_ARTFACTORY_CACHE_URL
"
${
JFROG_ARTFACTORY_URL
}
/milvus/thirdparty/cache/
${
CMAKE_OS_NAME
}
/
${
UBUNTU_VERSION
}
/
${
MILVUS_BUILD_ARCH
}
/
${
BUILD_TYPE
}
"
)
else
()
set
(
JFROG_ARTFACTORY_CACHE_URL
"
${
JFROG_ARTFACTORY_URL
}
/milvus/thirdparty/cache/
${
CMAKE_OS_NAME
}
/
${
MILVUS_BUILD_ARCH
}
/
${
BUILD_TYPE
}
"
)
endif
()
if
(
DEFINED ENV{JFROG_USER_NAME}
)
set
(
JFROG_USER_NAME
"$ENV{JFROG_USER_NAME}"
)
endif
()
if
(
NOT DEFINED JFROG_USER_NAME
)
endif
()
if
(
NOT DEFINED JFROG_USER_NAME
)
message
(
FATAL_ERROR
"JFROG_USER_NAME is not set"
)
endif
()
if
(
DEFINED ENV{JFROG_PASSWORD}
)
endif
()
if
(
DEFINED ENV{JFROG_PASSWORD}
)
set
(
JFROG_PASSWORD
"$ENV{JFROG_PASSWORD}"
)
endif
()
if
(
NOT DEFINED JFROG_PASSWORD
)
endif
()
if
(
NOT DEFINED JFROG_PASSWORD
)
message
(
FATAL_ERROR
"JFROG_PASSWORD is not set"
)
endif
()
endif
()
set
(
THIRDPARTY_PACKAGE_CACHE
"
${
THIRDPARTY_DIR
}
/cache"
)
if
(
NOT EXISTS
${
THIRDPARTY_PACKAGE_CACHE
}
)
if
(
NOT EXISTS
${
THIRDPARTY_PACKAGE_CACHE
}
)
message
(
STATUS
"Will create cached directory:
${
THIRDPARTY_PACKAGE_CACHE
}
"
)
file
(
MAKE_DIRECTORY
${
THIRDPARTY_PACKAGE_CACHE
}
)
endif
()
endif
()
endif
()
endif
()
macro
(
resolve_dependency DEPENDENCY_NAME
)
if
(
${
DEPENDENCY_NAME
}
_SOURCE STREQUAL
"AUTO"
)
...
...
@@ -182,13 +193,13 @@ set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
set
(
EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=
${
CMAKE_C_COMPILER
}
-DCMAKE_CXX_COMPILER=
${
CMAKE_CXX_COMPILER
}
)
if
(
CMAKE_AR
)
if
(
CMAKE_AR
)
set
(
EP_COMMON_TOOLCHAIN
${
EP_COMMON_TOOLCHAIN
}
-DCMAKE_AR=
${
CMAKE_AR
}
)
endif
()
endif
()
if
(
CMAKE_RANLIB
)
if
(
CMAKE_RANLIB
)
set
(
EP_COMMON_TOOLCHAIN
${
EP_COMMON_TOOLCHAIN
}
-DCMAKE_RANLIB=
${
CMAKE_RANLIB
}
)
endif
()
endif
()
# External projects are still able to override the following declarations.
# cmake command line will favor the last defined variable when a duplicate is
...
...
@@ -202,20 +213,20 @@ set(EP_COMMON_CMAKE_ARGS
-DCMAKE_CXX_FLAGS=
${
EP_CXX_FLAGS
}
-DCMAKE_CXX_FLAGS_
${
UPPERCASE_BUILD_TYPE
}
=
${
EP_CXX_FLAGS
}
)
if
(
NOT MILVUS_VERBOSE_THIRDPARTY_BUILD
)
if
(
NOT MILVUS_VERBOSE_THIRDPARTY_BUILD
)
set
(
EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1
)
else
()
else
()
set
(
EP_LOG_OPTIONS
)
endif
()
endif
()
# Ensure that a default make is set
if
(
"
${
MAKE
}
"
STREQUAL
""
)
if
(
"
${
MAKE
}
"
STREQUAL
""
)
find_program
(
MAKE make
)
endif
()
endif
()
if
(
NOT DEFINED MAKE_BUILD_ARGS
)
set
(
MAKE_BUILD_ARGS
"-j8"
)
endif
()
endif
()
message
(
STATUS
"Third Party MAKE_BUILD_ARGS =
${
MAKE_BUILD_ARGS
}
"
)
# ----------------------------------------------------------------------
...
...
@@ -230,39 +241,39 @@ find_package(Threads REQUIRED)
# Read toolchain versions from cpp/thirdparty/versions.txt
file
(
STRINGS
"
${
THIRDPARTY_DIR
}
/versions.txt"
TOOLCHAIN_VERSIONS_TXT
)
foreach
(
_VERSION_ENTRY
${
TOOLCHAIN_VERSIONS_TXT
}
)
foreach
(
_VERSION_ENTRY
${
TOOLCHAIN_VERSIONS_TXT
}
)
# Exclude comments
if
(
NOT _VERSION_ENTRY MATCHES
"^[^#][A-Za-z0-9-_]+_VERSION="
)
if
(
NOT _VERSION_ENTRY MATCHES
"^[^#][A-Za-z0-9-_]+_VERSION="
)
continue
()
endif
()
endif
()
string
(
REGEX MATCH
"^[^=]*"
_LIB_NAME
${
_VERSION_ENTRY
}
)
string
(
REPLACE
"
${
_LIB_NAME
}
="
""
_LIB_VERSION
${
_VERSION_ENTRY
}
)
# Skip blank or malformed lines
if
(
${
_LIB_VERSION
}
STREQUAL
""
)
if
(
${
_LIB_VERSION
}
STREQUAL
""
)
continue
()
endif
()
endif
()
# For debugging
#message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}")
set
(
${
_LIB_NAME
}
"
${
_LIB_VERSION
}
"
)
endforeach
()
endforeach
()
if
(
DEFINED ENV{MILVUS_GTEST_URL}
)
set
(
GTEST_SOURCE_URL
"$ENV{MILVUS_GTEST_URL}"
)
else
()
set
(
GTEST_SOURCE_URL
"https://github.com/google/googletest/archive/release-
${
GTEST_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
GTEST_MD5
"2e6fbeb6a91310a16efe181886c59596"
)
if
(
DEFINED ENV{MILVUS_MYSQLPP_URL}
)
if
(
DEFINED ENV{MILVUS_MYSQLPP_URL}
)
set
(
MYSQLPP_SOURCE_URL
"$ENV{MILVUS_MYSQLPP_URL}"
)
else
()
else
()
set
(
MYSQLPP_SOURCE_URL
"https://tangentsoft.com/mysqlpp/releases/mysql++-
${
MYSQLPP_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
MYSQLPP_MD5
"cda38b5ecc0117de91f7c42292dd1e79"
)
if
(
DEFINED ENV{MILVUS_PROMETHEUS_URL}
)
...
...
@@ -270,60 +281,60 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL})
else
()
set
(
PROMETHEUS_SOURCE_URL
https://github.com/jupp0r/prometheus-cpp.git
)
endif
()
endif
()
if
(
DEFINED ENV{MILVUS_SQLITE_URL}
)
if
(
DEFINED ENV{MILVUS_SQLITE_URL}
)
set
(
SQLITE_SOURCE_URL
"$ENV{MILVUS_SQLITE_URL}"
)
else
()
else
()
set
(
SQLITE_SOURCE_URL
"https://www.sqlite.org/2019/sqlite-autoconf-
${
SQLITE_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
SQLITE_MD5
"3c68eb400f8354605736cd55400e1572"
)
if
(
DEFINED ENV{MILVUS_SQLITE_ORM_URL}
)
if
(
DEFINED ENV{MILVUS_SQLITE_ORM_URL}
)
set
(
SQLITE_ORM_SOURCE_URL
"$ENV{MILVUS_SQLITE_ORM_URL}"
)
else
()
else
()
set
(
SQLITE_ORM_SOURCE_URL
"https://github.com/fnc12/sqlite_orm/archive/
${
SQLITE_ORM_VERSION
}
.zip"
)
endif
()
endif
()
set
(
SQLITE_ORM_MD5
"ba9a405a8a1421c093aa8ce988ff8598"
)
if
(
DEFINED ENV{MILVUS_YAMLCPP_URL}
)
if
(
DEFINED ENV{MILVUS_YAMLCPP_URL}
)
set
(
YAMLCPP_SOURCE_URL
"$ENV{MILVUS_YAMLCPP_URL}"
)
else
()
else
()
set
(
YAMLCPP_SOURCE_URL
"https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-
${
YAMLCPP_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
YAMLCPP_MD5
"5b943e9af0060d0811148b037449ef82"
)
if
(
DEFINED ENV{MILVUS_LIBUNWIND_URL}
)
if
(
DEFINED ENV{MILVUS_LIBUNWIND_URL}
)
set
(
LIBUNWIND_SOURCE_URL
"$ENV{MILVUS_LIBUNWIND_URL}"
)
else
()
else
()
set
(
LIBUNWIND_SOURCE_URL
"https://github.com/libunwind/libunwind/releases/download/v
${
LIBUNWIND_VERSION
}
/libunwind-
${
LIBUNWIND_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
LIBUNWIND_MD5
"a04f69d66d8e16f8bf3ab72a69112cd6"
)
if
(
DEFINED ENV{MILVUS_GPERFTOOLS_URL}
)
if
(
DEFINED ENV{MILVUS_GPERFTOOLS_URL}
)
set
(
GPERFTOOLS_SOURCE_URL
"$ENV{MILVUS_GPERFTOOLS_URL}"
)
else
()
else
()
set
(
GPERFTOOLS_SOURCE_URL
"https://github.com/gperftools/gperftools/releases/download/gperftools-
${
GPERFTOOLS_VERSION
}
/gperftools-
${
GPERFTOOLS_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
GPERFTOOLS_MD5
"c6a852a817e9160c79bdb2d3101b4601"
)
if
(
DEFINED ENV{MILVUS_GRPC_URL}
)
if
(
DEFINED ENV{MILVUS_GRPC_URL}
)
set
(
GRPC_SOURCE_URL
"$ENV{MILVUS_GRPC_URL}"
)
else
()
else
()
set
(
GRPC_SOURCE_URL
"https://github.com/youny626/grpc-milvus/archive/
${
GRPC_VERSION
}
.zip"
)
endif
()
endif
()
set
(
GRPC_MD5
"0362ba219f59432c530070b5f5c3df73"
)
if
(
DEFINED ENV{MILVUS_ZLIB_URL}
)
if
(
DEFINED ENV{MILVUS_ZLIB_URL}
)
set
(
ZLIB_SOURCE_URL
"$ENV{MILVUS_ZLIB_URL}"
)
else
()
else
()
set
(
ZLIB_SOURCE_URL
"https://github.com/madler/zlib/archive/
${
ZLIB_VERSION
}
.tar.gz"
)
endif
()
endif
()
set
(
ZLIB_MD5
"0095d2d2d1f3442ce1318336637b695f"
)
# ----------------------------------------------------------------------
...
...
@@ -334,13 +345,13 @@ macro(build_gtest)
set
(
GTEST_VENDORED TRUE
)
set
(
GTEST_CMAKE_CXX_FLAGS
"
${
EP_CXX_FLAGS
}
"
)
if
(
APPLE
)
if
(
APPLE
)
set
(
GTEST_CMAKE_CXX_FLAGS
${
GTEST_CMAKE_CXX_FLAGS
}
-DGTEST_USE_OWN_TR1_TUPLE=1
-Wno-unused-value
-Wno-ignored-attributes
)
endif
()
endif
()
set
(
GTEST_PREFIX
"
${
CMAKE_CURRENT_BINARY_DIR
}
/googletest_ep-prefix/src/googletest_ep"
)
set
(
GTEST_INCLUDE_DIR
"
${
GTEST_PREFIX
}
/include"
)
...
...
@@ -359,9 +370,9 @@ macro(build_gtest)
set
(
GMOCK_INCLUDE_DIR
"
${
GTEST_PREFIX
}
/include"
)
set
(
GMOCK_STATIC_LIB
"
${
GTEST_PREFIX
}
/lib/
${
CMAKE_STATIC_LIBRARY_PREFIX
}
gmock
${
CMAKE_STATIC_LIBRARY_SUFFIX
}
"
)
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
GTEST_CACHE_PACKAGE_NAME
"googletest_
${
GTEST_MD5
}
.tar.gz"
)
set
(
GTEST_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
GTEST_CACHE_PACKAGE_NAME
}
"
)
set
(
GTEST_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
GTEST_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -385,10 +396,10 @@ macro(build_gtest)
${
EP_LOG_OPTIONS
}
)
ExternalProject_Create_Cache
(
googletest_ep
${
GTEST_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/googletest_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
GTEST_CACHE_URL
}
)
else
()
else
()
ExternalProject_Use_Cache
(
googletest_ep
${
GTEST_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
else
()
endif
()
else
()
ExternalProject_Add
(
googletest_ep
URL
${
GTEST_SOURCE_URL
}
...
...
@@ -402,20 +413,20 @@ macro(build_gtest)
CMAKE_ARGS
${
GTEST_CMAKE_ARGS
}
${
EP_LOG_OPTIONS
}
)
endif
()
endif
()
# The include directory must exist before it is referenced by a target.
file
(
MAKE_DIRECTORY
"
${
GTEST_INCLUDE_DIR
}
"
)
add_library
(
gtest STATIC IMPORTED
)
set_target_properties
(
gtest
PROPERTIES IMPORTED_LOCATION
"
${
GTEST_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GTEST_INCLUDE_DIR
}
"
)
PROPERTIES IMPORTED_LOCATION
"
${
GTEST_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GTEST_INCLUDE_DIR
}
"
)
add_library
(
gtest_main STATIC IMPORTED
)
set_target_properties
(
gtest_main
PROPERTIES IMPORTED_LOCATION
"
${
GTEST_MAIN_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GTEST_INCLUDE_DIR
}
"
)
PROPERTIES IMPORTED_LOCATION
"
${
GTEST_MAIN_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GTEST_INCLUDE_DIR
}
"
)
add_library
(
gmock STATIC IMPORTED
)
set_target_properties
(
gmock
...
...
@@ -431,13 +442,13 @@ endmacro()
if
(
MILVUS_BUILD_TESTS
)
resolve_dependency
(
GTest
)
if
(
NOT GTEST_VENDORED
)
endif
()
if
(
NOT GTEST_VENDORED
)
endif
()
get_target_property
(
GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES
)
link_directories
(
SYSTEM
"
${
GTEST_PREFIX
}
/lib"
)
include_directories
(
SYSTEM
${
GTEST_INCLUDE_DIR
}
)
endif
()
endif
()
# ----------------------------------------------------------------------
# MySQL++
...
...
@@ -456,7 +467,7 @@ macro(build_mysqlpp)
"CXXFLAGS=
${
EP_CXX_FLAGS
}
"
"LDFLAGS=-pthread"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
MYSQLPP_CACHE_PACKAGE_NAME
"mysqlpp_
${
MYSQLPP_MD5
}
.tar.gz"
)
set
(
MYSQLPP_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
MYSQLPP_CACHE_PACKAGE_NAME
}
"
)
set
(
MYSQLPP_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
MYSQLPP_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -479,15 +490,15 @@ macro(build_mysqlpp)
${
MYSQLPP_SHARED_LIB
}
)
ExternalProject_Create_Cache
(
mysqlpp_ep
${
MYSQLPP_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/mysqlpp_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
MYSQLPP_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
MYSQLPP_CACHE_URL
}
${
MYSQLPP_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
MYSQLPP_CACHE_URL
}
TO
${
MYSQLPP_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
mysqlpp_ep
${
MYSQLPP_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
mysqlpp_ep
URL
${
MYSQLPP_SOURCE_URL
}
...
...
@@ -501,7 +512,7 @@ macro(build_mysqlpp)
1
BUILD_BYPRODUCTS
${
MYSQLPP_SHARED_LIB
}
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
MYSQLPP_INCLUDE_DIR
}
"
)
add_library
(
mysqlpp SHARED IMPORTED
)
...
...
@@ -515,13 +526,13 @@ macro(build_mysqlpp)
endmacro
()
if
(
MILVUS_WITH_MYSQLPP
)
if
(
MILVUS_WITH_MYSQLPP
)
resolve_dependency
(
MySQLPP
)
get_target_property
(
MYSQLPP_INCLUDE_DIR mysqlpp INTERFACE_INCLUDE_DIRECTORIES
)
include_directories
(
SYSTEM
"
${
MYSQLPP_INCLUDE_DIR
}
"
)
link_directories
(
SYSTEM
${
MYSQLPP_PREFIX
}
/lib
)
endif
()
endif
()
# ----------------------------------------------------------------------
# Prometheus
...
...
@@ -547,9 +558,9 @@ macro(build_prometheus)
"-DCMAKE_INSTALL_PREFIX=
${
PROMETHEUS_PREFIX
}
"
-DCMAKE_BUILD_TYPE=Release
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
execute_process
(
COMMAND sh -c
"git ls-remote --heads --tags
${
PROMETHEUS_SOURCE_URL
}
${
PROMETHEUS_VERSION
}
| cut -f 1"
OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID
)
if
(
${
PROMETHEUS_LAST_COMMIT_ID
}
MATCHES
"^[^#][a-z0-9]+"
)
if
(
${
PROMETHEUS_LAST_COMMIT_ID
}
MATCHES
"^[^#][a-z0-9]+"
)
string
(
MD5 PROMETHEUS_COMBINE_MD5
"
${
PROMETHEUS_LAST_COMMIT_ID
}
"
)
set
(
PROMETHEUS_CACHE_PACKAGE_NAME
"prometheus_
${
PROMETHEUS_COMBINE_MD5
}
.tar.gz"
)
set
(
PROMETHEUS_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
PROMETHEUS_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -583,18 +594,18 @@ macro(build_prometheus)
"
${
PROMETHEUS_PULL_STATIC_LIB
}
"
)
ExternalProject_Create_Cache
(
prometheus_ep
${
PROMETHEUS_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/prometheus_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
PROMETHEUS_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
PROMETHEUS_CACHE_URL
}
${
PROMETHEUS_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
PROMETHEUS_CACHE_URL
}
TO
${
PROMETHEUS_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
prometheus_ep
${
PROMETHEUS_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
message
(
FATAL_ERROR
"The last commit ID of
\"
${
PROMETHEUS_SOURCE_URL
}
\"
repository don't match!"
)
endif
()
else
()
endif
()
else
()
externalproject_add
(
prometheus_ep
GIT_REPOSITORY
${
PROMETHEUS_SOURCE_URL
}
...
...
@@ -618,7 +629,7 @@ macro(build_prometheus)
"
${
PROMETHEUS_CORE_STATIC_LIB
}
"
"
${
PROMETHEUS_PUSH_STATIC_LIB
}
"
"
${
PROMETHEUS_PULL_STATIC_LIB
}
"
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
PROMETHEUS_PREFIX
}
/push/include"
)
add_library
(
prometheus-cpp-push STATIC IMPORTED
)
...
...
@@ -642,7 +653,7 @@ macro(build_prometheus)
add_dependencies
(
prometheus-cpp-core prometheus_ep
)
endmacro
()
if
(
MILVUS_WITH_PROMETHEUS
)
if
(
MILVUS_WITH_PROMETHEUS
)
resolve_dependency
(
Prometheus
)
...
...
@@ -655,7 +666,7 @@ if(MILVUS_WITH_PROMETHEUS)
link_directories
(
SYSTEM
${
PROMETHEUS_PREFIX
}
/core/
)
include_directories
(
SYSTEM
${
PROMETHEUS_PREFIX
}
/core/include
)
endif
()
endif
()
# ----------------------------------------------------------------------
# SQLite
...
...
@@ -674,7 +685,7 @@ macro(build_sqlite)
"CFLAGS=
${
EP_C_FLAGS
}
"
"CXXFLAGS=
${
EP_CXX_FLAGS
}
"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
SQLITE_CACHE_PACKAGE_NAME
"sqlite_
${
SQLITE_MD5
}
.tar.gz"
)
set
(
SQLITE_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
SQLITE_CACHE_PACKAGE_NAME
}
"
)
set
(
SQLITE_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
SQLITE_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -698,15 +709,15 @@ macro(build_sqlite)
"
${
SQLITE_STATIC_LIB
}
"
)
ExternalProject_Create_Cache
(
sqlite_ep
${
SQLITE_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/sqlite_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
SQLITE_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
SQLITE_CACHE_URL
}
${
SQLITE_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
SQLITE_CACHE_URL
}
TO
${
SQLITE_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
sqlite_ep
${
SQLITE_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
sqlite_ep
URL
${
SQLITE_SOURCE_URL
}
...
...
@@ -721,7 +732,7 @@ macro(build_sqlite)
1
BUILD_BYPRODUCTS
"
${
SQLITE_STATIC_LIB
}
"
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
SQLITE_INCLUDE_DIR
}
"
)
add_library
(
sqlite STATIC IMPORTED
)
...
...
@@ -733,11 +744,11 @@ macro(build_sqlite)
add_dependencies
(
sqlite sqlite_ep
)
endmacro
()
if
(
MILVUS_WITH_SQLITE
)
if
(
MILVUS_WITH_SQLITE
)
resolve_dependency
(
SQLite
)
include_directories
(
SYSTEM
"
${
SQLITE_INCLUDE_DIR
}
"
)
link_directories
(
SYSTEM
${
SQLITE_PREFIX
}
/lib/
)
endif
()
endif
()
# ----------------------------------------------------------------------
# SQLite_ORM
...
...
@@ -753,16 +764,16 @@ macro(build_sqlite_orm)
file
(
DOWNLOAD
${
SQLITE_ORM_SOURCE_URL
}
${
SQLITE_ORM_TAR_NAME
}
)
execute_process
(
COMMAND
${
CMAKE_COMMAND
}
-E tar -xf
${
SQLITE_ORM_TAR_NAME
}
WORKING_DIRECTORY
${
SQLITE_ORM_PREFIX
}
)
WORKING_DIRECTORY
${
SQLITE_ORM_PREFIX
}
)
endif
()
endmacro
()
if
(
MILVUS_WITH_SQLITE_ORM
)
if
(
MILVUS_WITH_SQLITE_ORM
)
resolve_dependency
(
SQLite_ORM
)
include_directories
(
SYSTEM
"
${
SQLITE_ORM_INCLUDE_DIR
}
"
)
endif
()
endif
()
# ----------------------------------------------------------------------
# yaml-cpp
...
...
@@ -779,7 +790,7 @@ macro(build_yamlcpp)
-DYAML_CPP_BUILD_TESTS=OFF
-DYAML_CPP_BUILD_TOOLS=OFF
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
YAMLCPP_CACHE_PACKAGE_NAME
"yaml-cpp_
${
YAMLCPP_MD5
}
.tar.gz"
)
set
(
YAMLCPP_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
YAMLCPP_CACHE_PACKAGE_NAME
}
"
)
set
(
YAMLCPP_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
YAMLCPP_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -800,15 +811,15 @@ macro(build_yamlcpp)
${
YAMLCPP_CMAKE_ARGS
}
)
ExternalProject_Create_Cache
(
yaml-cpp_ep
${
YAMLCPP_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/yaml-cpp_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
YAMLCPP_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
YAMLCPP_CACHE_URL
}
${
YAMLCPP_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
YAMLCPP_CACHE_URL
}
TO
${
YAMLCPP_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
yaml-cpp_ep
${
YAMLCPP_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
yaml-cpp_ep
URL
${
YAMLCPP_SOURCE_URL
}
...
...
@@ -820,7 +831,7 @@ macro(build_yamlcpp)
"
${
YAMLCPP_STATIC_LIB
}
"
CMAKE_ARGS
${
YAMLCPP_CMAKE_ARGS
}
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
YAMLCPP_INCLUDE_DIR
}
"
)
add_library
(
yaml-cpp STATIC IMPORTED
)
...
...
@@ -831,13 +842,13 @@ macro(build_yamlcpp)
add_dependencies
(
yaml-cpp yaml-cpp_ep
)
endmacro
()
if
(
MILVUS_WITH_YAMLCPP
)
if
(
MILVUS_WITH_YAMLCPP
)
resolve_dependency
(
yaml-cpp
)
get_target_property
(
YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES
)
link_directories
(
SYSTEM
${
YAMLCPP_PREFIX
}
/lib/
)
include_directories
(
SYSTEM
${
YAMLCPP_INCLUDE_DIR
}
)
endif
()
endif
()
# ----------------------------------------------------------------------
# libunwind
...
...
@@ -849,7 +860,7 @@ macro(build_libunwind)
set
(
LIBUNWIND_SHARED_LIB
"
${
LIBUNWIND_PREFIX
}
/lib/libunwind
${
CMAKE_SHARED_LIBRARY_SUFFIX
}
"
)
set
(
LIBUNWIND_CONFIGURE_ARGS
"--prefix=
${
LIBUNWIND_PREFIX
}
"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
LIBUNWIND_CACHE_PACKAGE_NAME
"libunwind_
${
LIBUNWIND_MD5
}
.tar.gz"
)
set
(
LIBUNWIND_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
LIBUNWIND_CACHE_PACKAGE_NAME
}
"
)
set
(
LIBUNWIND_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
LIBUNWIND_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -874,15 +885,15 @@ macro(build_libunwind)
${
LIBUNWIND_SHARED_LIB
}
)
ExternalProject_Create_Cache
(
libunwind_ep
${
LIBUNWIND_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/libunwind_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
LIBUNWIND_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
LIBUNWIND_CACHE_URL
}
${
LIBUNWIND_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
LIBUNWIND_CACHE_URL
}
TO
${
LIBUNWIND_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
libunwind_ep
${
LIBUNWIND_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
libunwind_ep
URL
${
LIBUNWIND_SOURCE_URL
}
...
...
@@ -898,7 +909,7 @@ macro(build_libunwind)
${
MAKE
}
install
BUILD_BYPRODUCTS
${
LIBUNWIND_SHARED_LIB
}
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
LIBUNWIND_INCLUDE_DIR
}
"
)
...
...
@@ -910,12 +921,12 @@ macro(build_libunwind)
add_dependencies
(
libunwind libunwind_ep
)
endmacro
()
if
(
MILVUS_WITH_LIBUNWIND
)
if
(
MILVUS_WITH_LIBUNWIND
)
resolve_dependency
(
libunwind
)
get_target_property
(
LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES
)
include_directories
(
SYSTEM
${
LIBUNWIND_INCLUDE_DIR
}
)
endif
()
endif
()
# ----------------------------------------------------------------------
# gperftools
...
...
@@ -927,7 +938,7 @@ macro(build_gperftools)
set
(
GPERFTOOLS_STATIC_LIB
"
${
GPERFTOOLS_PREFIX
}
/lib/libprofiler
${
CMAKE_STATIC_LIBRARY_SUFFIX
}
"
)
set
(
GPERFTOOLS_CONFIGURE_ARGS
"--prefix=
${
GPERFTOOLS_PREFIX
}
"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
GPERFTOOLS_CACHE_PACKAGE_NAME
"gperftools_
${
GPERFTOOLS_MD5
}
.tar.gz"
)
set
(
GPERFTOOLS_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
GPERFTOOLS_CACHE_PACKAGE_NAME
}
"
)
set
(
GPERFTOOLS_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
GPERFTOOLS_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -952,15 +963,15 @@ macro(build_gperftools)
${
GPERFTOOLS_STATIC_LIB
}
)
ExternalProject_Create_Cache
(
gperftools_ep
${
GPERFTOOLS_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/gperftools_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
GPERFTOOLS_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
GPERFTOOLS_CACHE_URL
}
${
GPERFTOOLS_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
GPERFTOOLS_CACHE_URL
}
TO
${
GPERFTOOLS_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
gperftools_ep
${
GPERFTOOLS_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
gperftools_ep
URL
${
GPERFTOOLS_SOURCE_URL
}
...
...
@@ -976,7 +987,7 @@ macro(build_gperftools)
${
MAKE
}
install
BUILD_BYPRODUCTS
${
GPERFTOOLS_STATIC_LIB
}
)
endif
()
endif
()
ExternalProject_Add_StepDependencies
(
gperftools_ep build libunwind_ep
)
...
...
@@ -992,13 +1003,13 @@ macro(build_gperftools)
add_dependencies
(
gperftools libunwind_ep
)
endmacro
()
if
(
MILVUS_WITH_GPERFTOOLS
)
if
(
MILVUS_WITH_GPERFTOOLS
)
resolve_dependency
(
gperftools
)
get_target_property
(
GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES
)
include_directories
(
SYSTEM
${
GPERFTOOLS_INCLUDE_DIR
}
)
link_directories
(
SYSTEM
${
GPERFTOOLS_PREFIX
}
/lib
)
endif
()
endif
()
# ----------------------------------------------------------------------
# GRPC
...
...
@@ -1014,7 +1025,7 @@ macro(build_grpc)
set
(
GRPC_PROTOBUF_STATIC_LIB
"
${
GRPC_PROTOBUF_LIB_DIR
}
/
${
CMAKE_STATIC_LIBRARY_PREFIX
}
protobuf
${
CMAKE_STATIC_LIBRARY_SUFFIX
}
"
)
set
(
GRPC_PROTOC_STATIC_LIB
"
${
GRPC_PROTOBUF_LIB_DIR
}
/
${
CMAKE_STATIC_LIBRARY_PREFIX
}
protoc
${
CMAKE_STATIC_LIBRARY_SUFFIX
}
"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
GRPC_CACHE_PACKAGE_NAME
"grpc_
${
GRPC_MD5
}
.tar.gz"
)
set
(
GRPC_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
GRPC_CACHE_PACKAGE_NAME
}
"
)
set
(
GRPC_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
GRPC_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -1044,15 +1055,15 @@ macro(build_grpc)
ExternalProject_Add_StepDependencies
(
grpc_ep build zlib_ep
)
ExternalProject_Create_Cache
(
grpc_ep
${
GRPC_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/grpc_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
GRPC_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
GRPC_CACHE_URL
}
${
GRPC_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
GRPC_CACHE_URL
}
TO
${
GRPC_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
grpc_ep
${
GRPC_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
grpc_ep
URL
${
GRPC_SOURCE_URL
}
...
...
@@ -1074,7 +1085,7 @@ macro(build_grpc)
ExternalProject_Add_StepDependencies
(
grpc_ep build zlib_ep
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
GRPC_INCLUDE_DIR
}
"
)
...
...
@@ -1082,29 +1093,29 @@ macro(build_grpc)
set_target_properties
(
grpc
PROPERTIES IMPORTED_LOCATION
"
${
GRPC_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GRPC_INCLUDE_DIR
}
"
INTERFACE_LINK_LIBRARIES
"zlib"
)
INTERFACE_LINK_LIBRARIES
"zlib"
)
add_library
(
grpc++ STATIC IMPORTED
)
set_target_properties
(
grpc++
PROPERTIES IMPORTED_LOCATION
"
${
GRPC++_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GRPC_INCLUDE_DIR
}
"
INTERFACE_LINK_LIBRARIES
"zlib"
)
INTERFACE_LINK_LIBRARIES
"zlib"
)
add_library
(
grpcpp_channelz STATIC IMPORTED
)
set_target_properties
(
grpcpp_channelz
PROPERTIES IMPORTED_LOCATION
"
${
GRPCPP_CHANNELZ_STATIC_LIB
}
"
INTERFACE_INCLUDE_DIRECTORIES
"
${
GRPC_INCLUDE_DIR
}
"
INTERFACE_LINK_LIBRARIES
"zlib"
)
INTERFACE_LINK_LIBRARIES
"zlib"
)
add_library
(
grpc_protobuf STATIC IMPORTED
)
set_target_properties
(
grpc_protobuf
PROPERTIES IMPORTED_LOCATION
"
${
GRPC_PROTOBUF_STATIC_LIB
}
"
INTERFACE_LINK_LIBRARIES
"zlib"
)
INTERFACE_LINK_LIBRARIES
"zlib"
)
add_library
(
grpc_protoc STATIC IMPORTED
)
set_target_properties
(
grpc_protoc
PROPERTIES IMPORTED_LOCATION
"
${
GRPC_PROTOC_STATIC_LIB
}
"
INTERFACE_LINK_LIBRARIES
"zlib"
)
INTERFACE_LINK_LIBRARIES
"zlib"
)
add_dependencies
(
grpc grpc_ep
)
add_dependencies
(
grpc++ grpc_ep
)
...
...
@@ -1113,7 +1124,7 @@ macro(build_grpc)
add_dependencies
(
grpc_protoc grpc_ep
)
endmacro
()
if
(
MILVUS_WITH_GRPC
)
if
(
MILVUS_WITH_GRPC
)
resolve_dependency
(
GRPC
)
get_target_property
(
GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES
)
...
...
@@ -1123,7 +1134,7 @@ if(MILVUS_WITH_GRPC)
set
(
GRPC_THIRD_PARTY_DIR
${
CMAKE_CURRENT_BINARY_DIR
}
/grpc_ep-prefix/src/grpc_ep/third_party
)
include_directories
(
SYSTEM
${
GRPC_THIRD_PARTY_DIR
}
/protobuf/src
)
link_directories
(
SYSTEM
${
GRPC_PROTOBUF_LIB_DIR
}
)
endif
()
endif
()
# ----------------------------------------------------------------------
# zlib
...
...
@@ -1137,7 +1148,7 @@ macro(build_zlib)
set
(
ZLIB_CMAKE_ARGS
${
EP_COMMON_CMAKE_ARGS
}
"-DCMAKE_INSTALL_PREFIX=
${
ZLIB_PREFIX
}
"
-DBUILD_SHARED_LIBS=OFF
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
if
(
USE_JFROG_CACHE STREQUAL
"ON"
)
set
(
ZLIB_CACHE_PACKAGE_NAME
"zlib_
${
ZLIB_MD5
}
.tar.gz"
)
set
(
ZLIB_CACHE_URL
"
${
JFROG_ARTFACTORY_CACHE_URL
}
/
${
ZLIB_CACHE_PACKAGE_NAME
}
"
)
set
(
ZLIB_CACHE_PACKAGE_PATH
"
${
THIRDPARTY_PACKAGE_CACHE
}
/
${
ZLIB_CACHE_PACKAGE_NAME
}
"
)
...
...
@@ -1158,15 +1169,15 @@ macro(build_zlib)
${
ZLIB_CMAKE_ARGS
}
)
ExternalProject_Create_Cache
(
zlib_ep
${
ZLIB_CACHE_PACKAGE_PATH
}
"
${
CMAKE_CURRENT_BINARY_DIR
}
/zlib_ep-prefix"
${
JFROG_USER_NAME
}
${
JFROG_PASSWORD
}
${
ZLIB_CACHE_URL
}
)
else
()
else
()
file
(
DOWNLOAD
${
ZLIB_CACHE_URL
}
${
ZLIB_CACHE_PACKAGE_PATH
}
STATUS status
)
list
(
GET status 0 status_code
)
message
(
STATUS
"DOWNLOADING FROM
${
ZLIB_CACHE_URL
}
TO
${
ZLIB_CACHE_PACKAGE_PATH
}
. STATUS =
${
status_code
}
"
)
if
(
status_code EQUAL 0
)
ExternalProject_Use_Cache
(
zlib_ep
${
ZLIB_CACHE_PACKAGE_PATH
}
${
CMAKE_CURRENT_BINARY_DIR
}
)
endif
()
endif
()
else
()
endif
()
endif
()
else
()
externalproject_add
(
zlib_ep
URL
${
ZLIB_SOURCE_URL
}
...
...
@@ -1178,7 +1189,7 @@ macro(build_zlib)
"
${
ZLIB_STATIC_LIB
}
"
CMAKE_ARGS
${
ZLIB_CMAKE_ARGS
}
)
endif
()
endif
()
file
(
MAKE_DIRECTORY
"
${
ZLIB_INCLUDE_DIR
}
"
)
add_library
(
zlib STATIC IMPORTED
)
...
...
@@ -1189,9 +1200,9 @@ macro(build_zlib)
add_dependencies
(
zlib zlib_ep
)
endmacro
()
if
(
MILVUS_WITH_ZLIB
)
if
(
MILVUS_WITH_ZLIB
)
resolve_dependency
(
ZLIB
)
get_target_property
(
ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES
)
include_directories
(
SYSTEM
${
ZLIB_INCLUDE_DIR
}
)
endif
()
endif
()
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
9d95e472
...
...
@@ -258,7 +258,7 @@ Status
ExecutionEngineImpl
::
CopyToGpu
(
uint64_t
device_id
,
bool
hybrid
)
{
if
(
hybrid
)
{
const
std
::
string
key
=
location_
+
".quantizer"
;
std
::
vector
<
uint64_t
>
gpus
=
scheduler
::
get_gpu_pool
()
;
std
::
vector
<
uint64_t
>
gpus
{
device_id
}
;
const
int64_t
NOT_FOUND
=
-
1
;
int64_t
device_id
=
NOT_FOUND
;
...
...
core/src/index/cmake/ThirdPartyPackagesCore.cmake
浏览文件 @
9d95e472
...
...
@@ -245,7 +245,8 @@ if(CUSTOMIZATION)
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
set
(
FAISS_MD5
"f3b2ce3364c3fa7febd3aa7fdd0fe380"
)
# commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
set
(
FAISS_MD5
"bb30722c22390ce5f6759ccb216c1b2a"
)
# commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
endif
()
else
()
set
(
FAISS_SOURCE_URL
"https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz"
)
...
...
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
浏览文件 @
9d95e472
...
...
@@ -17,31 +17,40 @@
#include <gtest/gtest.h>
#include <hdf5.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <cassert>
#include <cmath>
#include <cstdio>
#include <vector>
#define USE_FAISS_V1_5_3 0
#if USE_FAISS_V1_5_3
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/utils.h>
#include <sys/stat.h>
#include <cstdlib>
#include <cstring>
#else
#include <faiss/gpu/GpuCloner.h>
#include <faiss/index_factory.h>
#include <faiss/utils/distances.h>
#endif
#include <faiss/AutoTune.h>
#include <faiss/Index.h>
#include <faiss/IndexIVF.h>
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/gpu/GpuIndexFlat.h>
#ifdef CUSTOMIZATION
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#endif
#include <faiss/gpu/StandardGpuResources.h>
#include <faiss/index_io.h>
#include <faiss/utils.h>
#include <hdf5.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <vector>
#ifdef CUSTOMIZATION
#include <faiss/gpu/GpuIndexIVFSQHybrid.h>
#endif
/*****************************************************
* To run this test, please download the HDF5 from
...
...
@@ -56,6 +65,8 @@ const char HDF5_DATASET_TEST[] = "test";
const
char
HDF5_DATASET_NEIGHBORS
[]
=
"neighbors"
;
const
char
HDF5_DATASET_DISTANCES
[]
=
"distances"
;
const
int32_t
GPU_DEVICE_IDX
=
0
;
enum
QueryMode
{
MODE_CPU
=
0
,
MODE_MIX
,
MODE_GPU
};
double
...
...
@@ -196,7 +207,7 @@ GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::i
for
(
int
i
=
0
;
i
<
nq
;
i
++
)
{
// count the num of results exist in ground truth result set
// each result replicates INDEX_ADD_LOOPS times
for
(
int
j_c
=
0
;
j_c
<
ground_
k
;
j_c
++
)
{
for
(
int
j_c
=
0
;
j_c
<
k
;
j_c
++
)
{
int
r_c
=
index
[
i
*
k
+
j_c
];
for
(
int
j_g
=
0
;
j_g
<
ground_k
/
index_add_loops
;
j_g
++
)
{
if
(
ground_index
[
i
*
ground_k
+
j_g
]
==
r_c
)
{
...
...
@@ -239,7 +250,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
double
t0
=
elapsed
();
const
std
::
string
ann_file_name
=
ann_test_name
+
HDF5_POSTFIX
;
const
int
GPU_DEVICE_IDX
=
0
;
faiss
::
Index
*
cpu_index
=
nullptr
,
*
gpu_index
=
nullptr
;
faiss
::
distance_compute_blas_threshold
=
800
;
...
...
@@ -249,30 +259,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
try
{
printf
(
"[%.3f s] Reading index file: %s
\n
"
,
elapsed
()
-
t0
,
index_file_name
.
c_str
());
cpu_index
=
faiss
::
read_index
(
index_file_name
.
c_str
());
if
(
mode
!=
MODE_CPU
)
{
faiss
::
gpu
::
GpuClonerOptions
option
;
option
.
allInGpu
=
true
;
faiss
::
IndexComposition
index_composition
;
index_composition
.
index
=
cpu_index
;
index_composition
.
quantizer
=
nullptr
;
switch
(
mode
)
{
case
MODE_CPU
:
assert
(
false
);
break
;
case
MODE_MIX
:
index_composition
.
mode
=
1
;
// 0: all data, 1: copy quantizer, 2: copy data
break
;
case
MODE_GPU
:
index_composition
.
mode
=
0
;
// 0: all data, 1: copy quantizer, 2: copy data
break
;
}
printf
(
"[%.3f s] Cloning CPU index to GPU
\n
"
,
elapsed
()
-
t0
);
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
&
index_composition
,
&
option
);
}
}
catch
(...)
{
size_t
nb
,
d
;
printf
(
"[%.3f s] Loading HDF5 file: %s
\n
"
,
elapsed
()
-
t0
,
ann_file_name
.
c_str
());
...
...
@@ -289,6 +275,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
printf
(
"[%.3f s] Cloning CPU index to GPU
\n
"
,
elapsed
()
-
t0
);
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
cpu_index
);
delete
cpu_index
;
printf
(
"[%.3f s] Training on %ld vectors
\n
"
,
elapsed
()
-
t0
,
nb
);
gpu_index
->
train
(
nb
,
xb
);
...
...
@@ -300,8 +287,9 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
}
printf
(
"[%.3f s] Coping GPU index to CPU
\n
"
,
elapsed
()
-
t0
);
delete
cpu_index
;
cpu_index
=
faiss
::
gpu
::
index_gpu_to_cpu
(
gpu_index
);
delete
gpu_index
;
faiss
::
IndexIVF
*
cpu_ivf_index
=
dynamic_cast
<
faiss
::
IndexIVF
*>
(
cpu_index
);
if
(
cpu_ivf_index
!=
nullptr
)
{
...
...
@@ -314,21 +302,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
delete
[]
xb
;
}
switch
(
mode
)
{
case
MODE_CPU
:
case
MODE_MIX
:
index
=
cpu_index
;
if
(
gpu_index
)
{
delete
gpu_index
;
}
break
;
case
MODE_GPU
:
index
=
gpu_index
;
if
(
cpu_index
)
{
delete
cpu_index
;
}
break
;
}
index
=
cpu_index
;
}
void
...
...
@@ -379,10 +353,12 @@ load_ground_truth(faiss::Index::idx_t*& gt, size_t& k, const std::string& ann_te
}
void
test_with_nprobes
(
const
std
::
string
&
ann_test_name
,
const
std
::
string
&
index_key
,
faiss
::
Index
*
index
,
test_with_nprobes
(
const
std
::
string
&
ann_test_name
,
const
std
::
string
&
index_key
,
faiss
::
Index
*
cpu_
index
,
faiss
::
gpu
::
StandardGpuResources
&
res
,
const
QueryMode
query_mode
,
const
faiss
::
Index
::
distance_t
*
xq
,
const
faiss
::
Index
::
idx_t
*
gt
,
const
std
::
vector
<
size_t
>
nprobes
,
const
int32_t
index_add_loops
,
const
int32_t
search_loops
)
{
double
t0
=
elapsed
();
const
size_t
NQ
=
1000
,
NQ_START
=
10
,
NQ_STEP
=
10
;
const
size_t
K
=
1000
,
K_START
=
100
,
K_STEP
=
10
;
const
size_t
GK
=
100
;
// topk of ground truth
...
...
@@ -390,6 +366,59 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
std
::
unordered_map
<
size_t
,
std
::
string
>
mode_str_map
=
{
{
MODE_CPU
,
"MODE_CPU"
},
{
MODE_MIX
,
"MODE_MIX"
},
{
MODE_GPU
,
"MODE_GPU"
}};
faiss
::
Index
*
gpu_index
,
*
index
;
if
(
query_mode
!=
MODE_CPU
)
{
faiss
::
gpu
::
GpuClonerOptions
option
;
option
.
allInGpu
=
true
;
faiss
::
IndexComposition
index_composition
;
index_composition
.
index
=
cpu_index
;
index_composition
.
quantizer
=
nullptr
;
double
copy_time
;
switch
(
query_mode
)
{
case
MODE_MIX
:
{
index_composition
.
mode
=
1
;
// 0: all data, 1: copy quantizer, 2: copy data
// warm up the transmission
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
&
index_composition
,
&
option
);
delete
gpu_index
;
copy_time
=
elapsed
();
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
&
index_composition
,
&
option
);
delete
gpu_index
;
copy_time
=
elapsed
()
-
copy_time
;
printf
(
"[%.3f s] Copy quantizer completed, cost %f s
\n
"
,
elapsed
()
-
t0
,
copy_time
);
auto
ivf_index
=
dynamic_cast
<
faiss
::
IndexIVF
*>
(
cpu_index
);
auto
is_gpu_flat_index
=
dynamic_cast
<
faiss
::
gpu
::
GpuIndexFlat
*>
(
ivf_index
->
quantizer
);
if
(
is_gpu_flat_index
==
nullptr
)
{
delete
ivf_index
->
quantizer
;
ivf_index
->
quantizer
=
index_composition
.
quantizer
;
}
index
=
cpu_index
;
break
;
}
case
MODE_GPU
:
index_composition
.
mode
=
0
;
// 0: all data, 1: copy quantizer, 2: copy data
// warm up the transmission
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
&
index_composition
,
&
option
);
delete
gpu_index
;
copy_time
=
elapsed
();
gpu_index
=
faiss
::
gpu
::
index_cpu_to_gpu
(
&
res
,
GPU_DEVICE_IDX
,
&
index_composition
,
&
option
);
copy_time
=
elapsed
()
-
copy_time
;
printf
(
"[%.3f s] Copy data completed, cost %f s
\n
"
,
elapsed
()
-
t0
,
copy_time
);
delete
cpu_index
;
index
=
gpu_index
;
break
;
}
}
else
{
index
=
cpu_index
;
}
for
(
auto
nprobe
:
nprobes
)
{
switch
(
query_mode
)
{
case
MODE_CPU
:
...
...
@@ -445,6 +474,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
delete
[]
I
;
delete
[]
D
;
}
delete
index
;
}
void
...
...
@@ -488,7 +519,6 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
delete
[]
xq
;
delete
[]
gt
;
delete
index
;
}
/************************************************************************************
...
...
@@ -520,8 +550,8 @@ TEST(FAISSTEST, BENCHMARK) {
#ifdef CUSTOMIZATION
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_MIX
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
// test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#endif
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
...
...
@@ -535,7 +565,7 @@ TEST(FAISSTEST, BENCHMARK) {
#ifdef CUSTOMIZATION
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_MIX
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
// test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#endif
}
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
浏览文件 @
9d95e472
...
...
@@ -55,16 +55,18 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
}
std
::
vector
<
uint64_t
>
gpus
=
scheduler
::
get_gpu_pool
();
std
::
vector
<
int64_t
>
all_free_mem
;
for
(
auto
&
gpu
:
gpus
)
{
auto
cache
=
cache
::
GpuCacheMgr
::
GetInstance
(
gpu
);
auto
free_mem
=
cache
->
CacheCapacity
()
-
cache
->
CacheUsage
();
all_free_mem
.
push_back
(
free_mem
);
}
auto
max_e
=
std
::
max_element
(
all_free_mem
.
begin
(),
all_free_mem
.
end
());
auto
best_index
=
std
::
distance
(
all_free_mem
.
begin
(),
max_e
);
auto
best_device_id
=
gpus
[
best_index
];
// std::vector<int64_t> all_free_mem;
// for (auto& gpu : gpus) {
// auto cache = cache::GpuCacheMgr::GetInstance(gpu);
// auto free_mem = cache->CacheCapacity() - cache->CacheUsage();
// all_free_mem.push_back(free_mem);
// }
//
// auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());
// auto best_index = std::distance(all_free_mem.begin(), max_e);
// auto best_device_id = gpus[best_index];
auto
best_device_id
=
count_
%
gpus
.
size
();
count_
++
;
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
ResourceType
::
GPU
,
best_device_id
);
if
(
not
res_ptr
)
{
...
...
core/src/scheduler/optimizer/LargeSQ8HPass.h
浏览文件 @
9d95e472
...
...
@@ -43,6 +43,7 @@ class LargeSQ8HPass : public Pass {
private:
int32_t
threshold_
=
std
::
numeric_limits
<
int32_t
>::
max
();
int64_t
count_
=
0
;
};
using
LargeSQ8HPassPtr
=
std
::
shared_ptr
<
LargeSQ8HPass
>
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录