Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

从无法访问的项目Fork

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 8464b8a2 编写于 作者: Z zhiru
浏览文件
操作

refactor cmake 


Former-commit-id: 587cc22f501fd12cbdaf2d9950fce3f2483c7020
上级 80dbaebd
隐藏空白更改
内联 并排
Showing with 225 addition and 1593 deletion
cpp/CHANGELOG.md
浏览文件 @ 8464b8a2
...@@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. ...@@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-556 - Add Job Definition in Scheduler - MS-556 - Add Job Definition in Scheduler
- MS-558 - Refine status code - MS-558 - Refine status code
- MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-562 - Add JobMgr and TaskCreator in Scheduler
- MS-566 - Refactor cmake
## New Feature ## New Feature
......
cpp/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -51,13 +51,6 @@ message(STATUS "Build type = ${BUILD_TYPE}") ...@@ -51,13 +51,6 @@ message(STATUS "Build type = ${BUILD_TYPE}")
project(milvus VERSION "${MILVUS_VERSION}") project(milvus VERSION "${MILVUS_VERSION}")
project(milvus_engine LANGUAGES CUDA CXX) project(milvus_engine LANGUAGES CUDA CXX)
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}") set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}") set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}") set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
...@@ -76,9 +69,6 @@ message(STATUS "Milvus version: " ...@@ -76,9 +69,6 @@ message(STATUS "Milvus version: "
"${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "
"(full: '${MILVUS_VERSION}')") "(full: '${MILVUS_VERSION}')")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA) find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
...@@ -105,8 +95,28 @@ else() ...@@ -105,8 +95,28 @@ else()
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
endif() endif()
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
find_path(MYSQL_INCLUDE_DIR
NAMES "mysql.h"
PATH_SUFFIXES "mysql")
if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
message(FATAL_ERROR "Could not found MySQL include directory")
else()
include_directories(${MYSQL_INCLUDE_DIR})
endif()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
include(ExternalProject) include(ExternalProject)
include(DefineOptions) include(DefineOptions)
include(BuildUtils) include(BuildUtils)
...@@ -114,7 +124,6 @@ include(ThirdPartyPackages) ...@@ -114,7 +124,6 @@ include(ThirdPartyPackages)
config_summary() config_summary()
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_subdirectory(src) add_subdirectory(src)
if (BUILD_UNIT_TEST STREQUAL "ON") if (BUILD_UNIT_TEST STREQUAL "ON")
......
cpp/cmake/DefineOptions.cmake
浏览文件 @ 8464b8a2
...@@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE ...@@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON) "Show output from ExternalProjects rather than just logging to files" ON)
define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF)
define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
Note that this requires linking Boost statically" ON) Note that this requires linking Boost statically" ON)
...@@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON) ...@@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON)
define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF)
define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF)
define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF)
define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON) define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON)
define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF)
define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF)
define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF)
define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON) define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON)
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON) define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
...@@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) ...@@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF)
#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON)
if(CMAKE_VERSION VERSION_LESS 3.7) if(CMAKE_VERSION VERSION_LESS 3.7)
set(MILVUS_WITH_ZSTD_DEFAULT OFF) set(MILVUS_WITH_ZSTD_DEFAULT OFF)
else() else()
...@@ -106,13 +88,13 @@ else() ...@@ -106,13 +88,13 @@ else()
endif() endif()
define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT}) define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON") if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif() endif()
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
#---------------------------------------------------------------------- #----------------------------------------------------------------------
if(MSVC) if(MSVC)
set_option_category("MSVC") set_option_category("MSVC")
......
cpp/cmake/ThirdPartyPackages.cmake
浏览文件 @ 8464b8a2
...@@ -16,27 +16,18 @@ ...@@ -16,27 +16,18 @@
set(MILVUS_THIRDPARTY_DEPENDENCIES set(MILVUS_THIRDPARTY_DEPENDENCIES
ARROW
BOOST BOOST
BZip2 BZip2
FAISS
GTest GTest
Knowhere
JSONCONS
LAPACK
Lz4 Lz4
MySQLPP MySQLPP
OpenBLAS
Prometheus Prometheus
RocksDB
Snappy Snappy
SQLite SQLite
SQLite_ORM SQLite_ORM
Thrift
yaml-cpp yaml-cpp
ZLIB ZLIB
ZSTD ZSTD
AWS
libunwind libunwind
gperftools gperftools
GRPC) GRPC)
...@@ -51,30 +42,16 @@ foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) ...@@ -51,30 +42,16 @@ foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES})
endforeach() endforeach()
macro(build_dependency DEPENDENCY_NAME) macro(build_dependency DEPENDENCY_NAME)
if("${DEPENDENCY_NAME}" STREQUAL "ARROW") if("${DEPENDENCY_NAME}" STREQUAL "BZip2")
build_arrow()
elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2")
build_bzip2() build_bzip2()
elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
build_gtest() build_gtest()
elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK")
build_lapack()
elseif("${DEPENDENCY_NAME}" STREQUAL "Knowhere")
build_knowhere()
elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4")
build_lz4() build_lz4()
elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP") elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP")
build_mysqlpp() build_mysqlpp()
elseif ("${DEPENDENCY_NAME}" STREQUAL "JSONCONS")
build_jsoncons()
elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus")
build_prometheus() build_prometheus()
elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB")
build_rocksdb()
elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy")
build_snappy() build_snappy()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
...@@ -87,8 +64,6 @@ macro(build_dependency DEPENDENCY_NAME) ...@@ -87,8 +64,6 @@ macro(build_dependency DEPENDENCY_NAME)
build_zlib() build_zlib()
elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD")
build_zstd() build_zstd()
elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
build_aws()
elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind")
build_libunwind() build_libunwind()
elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools")
...@@ -280,8 +255,7 @@ if(DEFINED ENV{MILVUS_BOOST_URL}) ...@@ -280,8 +255,7 @@ if(DEFINED ENV{MILVUS_BOOST_URL})
else() else()
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
set(BOOST_SOURCE_URL set(BOOST_SOURCE_URL
"http://192.168.1.201/artifactory/generic-local/tools/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz") "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz")
# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
endif() endif()
set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26") set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26")
...@@ -292,22 +266,6 @@ else() ...@@ -292,22 +266,6 @@ else()
endif() endif()
set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b")
if(DEFINED ENV{MILVUS_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}")
else()
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz")
endif()
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
set(FAISS_MD5 "31167ecbd1903fec600dc4ac00b9be9e")
if(DEFINED ENV{MILVUS_KNOWHERE_URL})
set(KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}")
else()
set(KNOWHERE_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/knowhere")
endif()
if (DEFINED ENV{MILVUS_GTEST_URL}) if (DEFINED ENV{MILVUS_GTEST_URL})
set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}") set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}")
else () else ()
...@@ -316,20 +274,6 @@ else () ...@@ -316,20 +274,6 @@ else ()
endif() endif()
set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596")
if (DEFINED ENV{MILVUS_JSONCONS_URL})
set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}")
else ()
set(JSONCONS_SOURCE_URL
"https://github.com/danielaparker/jsoncons/archive/v${JSONCONS_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MILVUS_LAPACK_URL})
set(LAPACK_SOURCE_URL "$ENV{MILVUS_LAPACK_URL}")
else()
set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz")
endif()
set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2")
if(DEFINED ENV{MILVUS_LZ4_URL}) if(DEFINED ENV{MILVUS_LZ4_URL})
set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}") set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}")
else() else()
...@@ -344,14 +288,6 @@ else() ...@@ -344,14 +288,6 @@ else()
endif() endif()
set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79")
if (DEFINED ENV{MILVUS_OPENBLAS_URL})
set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}")
else ()
set(OPENBLAS_SOURCE_URL
"https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz")
endif()
set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495")
if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) if (DEFINED ENV{MILVUS_PROMETHEUS_URL})
set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}")
else () else ()
...@@ -359,15 +295,6 @@ else () ...@@ -359,15 +295,6 @@ else ()
https://github.com/jupp0r/prometheus-cpp.git) https://github.com/jupp0r/prometheus-cpp.git)
endif() endif()
if (DEFINED ENV{MILVUS_ROCKSDB_URL})
set(ROCKSDB_SOURCE_URL "$ENV{MILVUS_ROCKSDB_URL}")
else ()
set(ROCKSDB_SOURCE_URL
"https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz")
endif()
set(ROCKSDB_MD5 "a8f2f594182e97a08629bcc66dfd3fa0")
if(DEFINED ENV{MILVUS_SNAPPY_URL}) if(DEFINED ENV{MILVUS_SNAPPY_URL})
set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}") set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}")
else() else()
...@@ -413,13 +340,6 @@ else() ...@@ -413,13 +340,6 @@ else()
endif() endif()
set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120") set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120")
if(DEFINED ENV{MILVUS_AWS_URL})
set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}")
else()
set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
endif()
set(AWS_MD5 "9217f5bc8bf23dea04f4466521c85fd9")
if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) if(DEFINED ENV{MILVUS_LIBUNWIND_URL})
set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
else() else()
...@@ -445,120 +365,6 @@ endif() ...@@ -445,120 +365,6 @@ endif()
set(GRPC_MD5 "7ec59ad54c85a12dcbbfede09bf413a9") set(GRPC_MD5 "7ec59ad54c85a12dcbbfede09bf413a9")
# ----------------------------------------------------------------------
# ARROW
macro(build_arrow)
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
set(ARROW_STATIC_LIB_NAME arrow)
set(ARROW_STATIC_LIB
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DARROW_BUILD_STATIC=ON
-DARROW_BUILD_SHARED=OFF
-DARROW_PARQUET=ON
-DARROW_USE_GLOG=OFF
-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
"-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs"
-DCMAKE_BUILD_TYPE=Release)
if(USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}")
set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${ARROW_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
else()
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!")
endif()
else()
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
endif()
file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
add_library(arrow STATIC IMPORTED)
set_target_properties(arrow
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
add_dependencies(arrow arrow_ep)
set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
add_custom_command(TARGET arrow_ep POST_BUILD
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/
DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
endmacro()
if(MILVUS_WITH_ARROW)
resolve_dependency(ARROW)
link_directories(SYSTEM ${ARROW_PREFIX}/lib/)
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating)) # Add Boost dependencies (code adapted from Apache Kudu (incubating))
...@@ -755,370 +561,6 @@ if(MILVUS_WITH_BZ2) ...@@ -755,370 +561,6 @@ if(MILVUS_WITH_BZ2)
include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}")
endif() endif()
# ----------------------------------------------------------------------
# Knowhere
macro(build_knowhere)
message(STATUS "Building knowhere from source")
set(KNOWHERE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/knowhere_ep-prefix/src/knowhere_ep")
set(KNOWHERE_INCLUDE_DIR "${KNOWHERE_PREFIX}/include")
set(KNOWHERE_STATIC_LIB
"${KNOWHERE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}knowhere${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(KNOWHERE_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${KNOWHERE_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib
"-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
"-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}"
-DCMAKE_BUILD_TYPE=Release)
externalproject_add(knowhere_ep
URL
${KNOWHERE_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${KNOWHERE_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${KNOWHERE_STATIC_LIB})
file(MAKE_DIRECTORY "${KNOWHERE_INCLUDE_DIR}")
add_library(knowhere STATIC IMPORTED)
set_target_properties(
knowhere
PROPERTIES IMPORTED_LOCATION "${KNOWHERE_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${KNOWHERE_INCLUDE_DIR}")
add_dependencies(knowhere knowhere_ep)
endmacro()
if(MILVUS_WITH_KNOWHERE)
resolve_dependency(Knowhere)
get_target_property(KNOWHERE_INCLUDE_DIR knowhere INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM "${KNOWHERE_PREFIX}/lib")
include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}")
include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService")
endif()
# ----------------------------------------------------------------------
# OpenBLAS
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
if(USE_JFROG_CACHE STREQUAL "ON")
set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz")
set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}")
set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${OPENBLAS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${OPENBLAS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_COMMAND
${MAKE}
PREFIX=${OPENBLAS_PREFIX}
install
BUILD_BYPRODUCTS
${OPENBLAS_STATIC_LIB})
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
else()
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_COMMAND
${MAKE}
PREFIX=${OPENBLAS_PREFIX}
install
BUILD_BYPRODUCTS
${OPENBLAS_STATIC_LIB})
endif()
file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
add_library(openblas STATIC IMPORTED)
set_target_properties(
openblas
PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
add_dependencies(openblas openblas_ep)
endmacro()
# ----------------------------------------------------------------------
# LAPACK
macro(build_lapack)
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
set(LAPACK_STATIC_LIB
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(LAPACK_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib)
if(USE_JFROG_CACHE STREQUAL "ON")
set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz")
set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}")
set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${LAPACK_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${LAPACK_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(lapack_ep
URL
${LAPACK_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${LAPACK_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${LAPACK_STATIC_LIB})
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
else()
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(lapack_ep
URL
${LAPACK_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${LAPACK_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${LAPACK_STATIC_LIB})
endif()
file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}")
add_library(lapack STATIC IMPORTED)
set_target_properties(
lapack
PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}")
add_dependencies(lapack lapack_ep)
endmacro()
# ----------------------------------------------------------------------
# FAISS
if(NOT DEFINED BUILD_FAISS_WITH_MKL)
set(BUILD_FAISS_WITH_MKL OFF)
endif()
if(EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL OFF)
endif()
endif()
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}"
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS}"
--without-python)
set(FAISS_CFLAGS ${EP_C_FLAGS})
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
message(STATUS "Build Faiss with MKL")
if(NOT DEFINED MKL_LIB_PATH)
set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64")
message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}")
endif()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
"LDFLAGS=-L${MKL_LIB_PATH}"
"LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl")
else()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib")
endif()
if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_75,code=sm_75"
)
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
endif()
if(USE_JFROG_CACHE STREQUAL "ON")
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz")
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif()
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
else()
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif()
endif()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss SHARED IMPORTED)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a)
set_target_properties(
faiss
PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" )
else()
set_target_properties(
faiss
PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "openblas;lapack" )
endif()
add_dependencies(faiss faiss_ep)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
add_dependencies(faiss openblas_ep)
add_dependencies(faiss lapack_ep)
endif()
endmacro()
if(MILVUS_WITH_FAISS)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
resolve_dependency(LAPACK)
get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
link_directories(SYSTEM "${LAPACK_PREFIX}/lib")
endif()
resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# Google gtest # Google gtest
...@@ -1232,30 +674,6 @@ if (MILVUS_BUILD_TESTS) ...@@ -1232,30 +674,6 @@ if (MILVUS_BUILD_TESTS)
include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif() endif()
# ----------------------------------------------------------------------
# JSONCONS
macro(build_jsoncons)
message(STATUS "Building JSONCONS-${JSONCONS_VERSION} from source")
set(JSONCONS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jsoncons_ep-prefix")
set(JSONCONS_TAR_NAME "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}.tar.gz")
set(JSONCONS_INCLUDE_DIR "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}/include")
if (NOT EXISTS ${JSONCONS_INCLUDE_DIR})
file(MAKE_DIRECTORY ${JSONCONS_PREFIX})
file(DOWNLOAD ${JSONCONS_SOURCE_URL}
${JSONCONS_TAR_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${JSONCONS_TAR_NAME}
WORKING_DIRECTORY ${JSONCONS_PREFIX})
endif ()
endmacro()
if(MILVUS_WITH_JSONCONS)
resolve_dependency(JSONCONS)
include_directories(SYSTEM "${JSONCONS_INCLUDE_DIR}")
endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# lz4 # lz4
...@@ -1563,95 +981,6 @@ if(MILVUS_WITH_PROMETHEUS) ...@@ -1563,95 +981,6 @@ if(MILVUS_WITH_PROMETHEUS)
endif() endif()
# ----------------------------------------------------------------------
# RocksDB
macro(build_rocksdb)
message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source")
set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep")
set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include")
set(ROCKSDB_STATIC_LIB_NAME rocksdb)
set(ROCKSDB_STATIC_LIB
"${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
if(USE_JFROG_CACHE STREQUAL "ON")
string(MD5 ROCKSDB_COMBINE_MD5 "${ROCKSDB_LAST_COMMIT_ID}")
set(ROCKSDB_CACHE_PACKAGE_NAME "rocksdb_${ROCKSDB_MD5}.tar.gz")
set(ROCKSDB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ROCKSDB_CACHE_PACKAGE_NAME}")
set(ROCKSDB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ROCKSDB_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${ROCKSDB_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ROCKSDB_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
ExternalProject_Create_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ROCKSDB_CACHE_URL})
else()
file(DOWNLOAD ${ROCKSDB_CACHE_URL} ${ROCKSDB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ROCKSDB_CACHE_URL} TO ${ROCKSDB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
endif()
file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include")
add_library(rocksdb STATIC IMPORTED)
set_target_properties(rocksdb
PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}")
add_dependencies(rocksdb rocksdb_ep)
endmacro()
if(MILVUS_WITH_ROCKSDB)
resolve_dependency(RocksDB)
link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/)
include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS})
endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# Snappy # Snappy
...@@ -2091,116 +1420,6 @@ if(MILVUS_WITH_ZSTD) ...@@ -2091,116 +1420,6 @@ if(MILVUS_WITH_ZSTD)
include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
endif() endif()
# ----------------------------------------------------------------------
# aws
macro(build_aws)
message(STATUS "Building aws-${AWS_VERSION} from source")
set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep")
set(AWS_CMAKE_ARGS
${EP_COMMON_TOOLCHAIN}
"-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}"
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_LIBDIR=lib
-DBUILD_ONLY=s3
-DBUILD_SHARED_LIBS=off
-DENABLE_TESTING=off
-DENABLE_UNITY_BUILD=on
-DNO_ENCRYPTION=off)
set(AWS_CPP_SDK_CORE_STATIC_LIB
"${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(AWS_CPP_SDK_S3_STATIC_LIB
"${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include")
set(AWS_CMAKE_ARGS
${AWS_CMAKE_ARGS}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS})
if(USE_JFROG_CACHE STREQUAL "ON")
set(AWS_CACHE_PACKAGE_NAME "aws_${AWS_MD5}.tar.gz")
set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}")
set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${AWS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${AWS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(aws_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
${AWS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_DIR
${AWS_PREFIX}
URL
${AWS_SOURCE_URL}
BUILD_BYPRODUCTS
"${AWS_CPP_SDK_S3_STATIC_LIB}"
"${AWS_CPP_SDK_CORE_STATIC_LIB}")
ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL})
else()
file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(aws_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
${AWS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_DIR
${AWS_PREFIX}
URL
${AWS_SOURCE_URL}
BUILD_BYPRODUCTS
"${AWS_CPP_SDK_S3_STATIC_LIB}"
"${AWS_CPP_SDK_CORE_STATIC_LIB}")
endif()
file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}")
add_library(aws-cpp-sdk-s3 STATIC IMPORTED)
set_target_properties(aws-cpp-sdk-s3
PROPERTIES
IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a")
add_library(aws-cpp-sdk-core STATIC IMPORTED)
set_target_properties(aws-cpp-sdk-core
PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a")
add_dependencies(aws-cpp-sdk-s3 aws_ep)
add_dependencies(aws-cpp-sdk-core aws_ep)
endmacro()
if(MILVUS_WITH_AWS)
resolve_dependency(AWS)
link_directories(SYSTEM ${AWS_PREFIX}/lib)
get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${AWS_CPP_SDK_S3_INCLUDE_DIR})
get_target_property(AWS_CPP_SDK_CORE_INCLUDE_DIR aws-cpp-sdk-core INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR})
endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# libunwind # libunwind
...@@ -2465,12 +1684,14 @@ macro(build_grpc) ...@@ -2465,12 +1684,14 @@ macro(build_grpc)
add_dependencies(grpc_protoc grpc_ep) add_dependencies(grpc_protoc grpc_ep)
endmacro() endmacro()
resolve_dependency(GRPC) if(MILVUS_WITH_GRPC)
resolve_dependency(GRPC)
get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
link_directories(SYSTEM ${GRPC_PREFIX}/lib) link_directories(SYSTEM ${GRPC_PREFIX}/lib)
set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
endif()
cpp/src/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,25 +17,41 @@ ...@@ -17,25 +17,41 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
add_subdirectory(core) add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS}) foreach (dir ${CORE_INCLUDE_DIRS})
include_directories(${dir}) include_directories(${dir})
endforeach() endforeach ()
aux_source_directory(cache cache_files) aux_source_directory(cache cache_files)
aux_source_directory(config config_files) aux_source_directory(config config_files)
aux_source_directory(server server_files)
aux_source_directory(server/grpc_impl grpcserver_files)
aux_source_directory(utils utils_files)
aux_source_directory(db db_main_files) aux_source_directory(db db_main_files)
aux_source_directory(db/engine db_engine_files) aux_source_directory(db/engine db_engine_files)
aux_source_directory(db/insert db_insert_files) aux_source_directory(db/insert db_insert_files)
aux_source_directory(db/meta db_meta_files) aux_source_directory(db/meta db_meta_files)
aux_source_directory(db/scheduler scheduler_files)
aux_source_directory(db/scheduler/context scheduler_context_files)
aux_source_directory(db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(grpc_service_files
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
aux_source_directory(metrics metrics_files) aux_source_directory(metrics metrics_files)
aux_source_directory(wrapper/knowhere knowhere_files)
aux_source_directory(scheduler/action scheduler_action_files) aux_source_directory(scheduler/action scheduler_action_files)
aux_source_directory(scheduler/event scheduler_event_files) aux_source_directory(scheduler/event scheduler_event_files)
...@@ -43,7 +59,7 @@ aux_source_directory(scheduler/job scheduler_job_files) ...@@ -43,7 +59,7 @@ aux_source_directory(scheduler/job scheduler_job_files)
aux_source_directory(scheduler/resource scheduler_resource_files) aux_source_directory(scheduler/resource scheduler_resource_files)
aux_source_directory(scheduler/task scheduler_task_files) aux_source_directory(scheduler/task scheduler_task_files)
aux_source_directory(scheduler scheduler_root_files) aux_source_directory(scheduler scheduler_root_files)
set(scheduler_srcs set(scheduler_files
${scheduler_action_files} ${scheduler_action_files}
${scheduler_event_files} ${scheduler_event_files}
${scheduler_job_files} ${scheduler_job_files}
...@@ -52,32 +68,12 @@ set(scheduler_srcs ...@@ -52,32 +68,12 @@ set(scheduler_srcs
${scheduler_root_files} ${scheduler_root_files}
) )
aux_source_directory(db/scheduler scheduler_files) aux_source_directory(server server_files)
aux_source_directory(db/scheduler/context scheduler_context_files) aux_source_directory(server/grpc_impl grpc_server_files)
aux_source_directory(db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(license_check_files
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
set(license_generator_files aux_source_directory(utils utils_files)
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
set(grpc_service_files aux_source_directory(wrapper/knowhere knowhere_files)
metrics/SystemInfo.cpp
metrics/SystemInfo.h
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
scheduler/Utils.h)
set(db_files set(db_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
...@@ -88,17 +84,12 @@ set(db_files ...@@ -88,17 +84,12 @@ set(db_files
${db_meta_files} ${db_meta_files}
${db_scheduler_files} ${db_scheduler_files}
${metrics_files} ${metrics_files}
${knowhere_files}
${utils_files} ${utils_files}
${knowhere_files}
) )
set(s3_client_files
storage/s3/S3ClientWrapper.cpp
storage/s3/S3ClientWrapper.h)
include_directories(/usr/include) include_directories(/usr/include)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
include_directories(/usr/include/mysql)
include_directories(grpc/gen-status) include_directories(grpc/gen-status)
include_directories(grpc/gen-milvus) include_directories(grpc/gen-milvus)
...@@ -109,17 +100,22 @@ set(client_grpc_lib ...@@ -109,17 +100,22 @@ set(client_grpc_lib
grpc_protobuf grpc_protobuf
grpc_protoc) grpc_protoc)
set(third_party_libs set(prometheus_lib
knowhere
sqlite
${client_grpc_lib}
yaml-cpp
prometheus-cpp-push prometheus-cpp-push
prometheus-cpp-pull prometheus-cpp-pull
prometheus-cpp-core prometheus-cpp-core)
set(boost_lib
boost_system_static boost_system_static
boost_filesystem_static boost_filesystem_static
boost_serialization_static boost_serialization_static)
set(third_party_libs
sqlite
${client_grpc_lib}
yaml-cpp
${prometheus_lib}
${boost_lib}
bzip2 bzip2
lz4 lz4
snappy snappy
...@@ -131,13 +127,12 @@ set(third_party_libs ...@@ -131,13 +127,12 @@ set(third_party_libs
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart cudart
) )
if (MILVUS_ENABLE_PROFILING STREQUAL "ON") if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs} set(third_party_libs ${third_party_libs}
gperftools gperftools
libunwind) libunwind
endif() )
endif ()
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(engine_libs set(engine_libs
...@@ -149,12 +144,11 @@ set(engine_libs ...@@ -149,12 +144,11 @@ set(engine_libs
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(engine_libs set(engine_libs
${engine_libs} ${engine_libs}
libquadmath.a libquadmath.a
) )
endif () endif ()
cuda_add_library(milvus_engine STATIC ${db_files}) cuda_add_library(milvus_engine STATIC ${db_files})
target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
...@@ -162,9 +156,7 @@ add_library(metrics STATIC ${metrics_files}) ...@@ -162,9 +156,7 @@ add_library(metrics STATIC ${metrics_files})
set(metrics_lib set(metrics_lib
yaml-cpp yaml-cpp
prometheus-cpp-push ${prometheus_lib}
prometheus-cpp-pull
prometheus-cpp-core
) )
target_link_libraries(metrics ${metrics_lib}) target_link_libraries(metrics ${metrics_lib})
...@@ -176,21 +168,17 @@ set(server_libs ...@@ -176,21 +168,17 @@ set(server_libs
metrics metrics
) )
set(knowhere_libs
knowhere
)
add_executable(milvus_server add_executable(milvus_server
${config_files} ${config_files}
${metrics_files}
${scheduler_files}
${server_files} ${server_files}
${grpcserver_files} ${grpc_server_files}
${utils_files}
${grpc_service_files} ${grpc_service_files}
${metrics_files} ${utils_files}
${scheduler_srcs}
) )
target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs}) target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs})
install(TARGETS milvus_server DESTINATION bin) install(TARGETS milvus_server DESTINATION bin)
......
cpp/src/sdk/examples/grpcsimple/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -20,11 +20,6 @@ ...@@ -20,11 +20,6 @@
aux_source_directory(src src_files) aux_source_directory(src src_files)
include_directories(src)
include_directories(../../include)
link_directories(${CMAKE_BINARY_DIR})
add_executable(sdk_simple add_executable(sdk_simple
main.cpp main.cpp
${src_files} ${src_files}
......
cpp/src/storage/IStorage.h 已删除 100644 → 0
浏览文件 @ 80dbaebd
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "db/Status.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace storage {
class IStorage {
public:
virtual Status Create(const std::string &ip_address,
const std::string &port,
const std::string &access_key,
const std::string &secret_key) = 0;
virtual Status Close() = 0;
virtual Status CreateBucket(std::string& bucket_name) = 0;
virtual Status DeleteBucket(std::string& bucket_name) = 0;
virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0;
};
}
}
}
}
\ No newline at end of file
cpp/src/storage/s3/S3ClientWrapper.cpp 已删除 100644 → 0
浏览文件 @ 80dbaebd
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//#include "S3ClientWrapper.h"
//
//#include <aws/s3/model/CreateBucketRequest.h>
//#include <aws/s3/model/DeleteBucketRequest.h>
//#include <aws/s3/model/PutObjectRequest.h>
//#include <aws/s3/model/GetObjectRequest.h>
//#include <aws/s3/model/DeleteObjectRequest.h>
//
//#include <iostream>
//#include <fstream>
//
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//Status
//S3ClientWrapper::Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) {
// Aws::InitAPI(options_);
// Aws::Client::ClientConfiguration cfg;
//
// // TODO: ip_address need to be validated.
//
// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
// cfg.scheme = Aws::Http::Scheme::HTTP;
// cfg.verifySSL =
// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
// client_ =
// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
// cfg,
// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
// false);
// if (client_ == nullptr) {
// std::string error = "Can't connect server.";
// return Status::Error(error);
// } else {
// return Status::OK();
// }
//}
//
//
//Status
//S3ClientWrapper::Close() {
// if (client_ != nullptr) {
// delete client_;
// client_ = nullptr;
// }
// Aws::ShutdownAPI(options_);
// return Status::OK();
//}
//
//Status
//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
// Aws::S3::Model::CreateBucketRequest request;
// request.SetBucket(bucket_name);
//
// auto outcome = client_->CreateBucket(request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "CreateBucket error: "
// << outcome.GetError().GetExceptionName() << std::endl
// << outcome.GetError().GetMessage() << std::endl;
// switch(outcome.GetError().GetErrorType()) {
// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
// return Status::AlreadyExist(outcome.GetError().GetMessage());
// default:
// return Status::Error(outcome.GetError().GetMessage());
// }
// }
//}
//
//Status
//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
// Aws::S3::Model::DeleteBucketRequest bucket_request;
// bucket_request.SetBucket(bucket_name);
//
// auto outcome = client_->DeleteBucket(bucket_request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "DeleteBucket error: "
// << outcome.GetError().GetExceptionName() << " - "
// << outcome.GetError().GetMessage() << std::endl;
// return Status::Error(outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
//
// PutObjectRequest putObjectRequest;
// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
//
// auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
// pathkey.c_str(),
// std::ios_base::in | std::ios_base::binary);
// putObjectRequest.SetBody(input_data);
// auto put_object_result = client_->PutObject(putObjectRequest);
// if (put_object_result.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
// << put_object_result.GetError().GetMessage() << std::endl;
// return Status::Error(put_object_result.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
// GetObjectRequest object_request;
// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
// auto get_object_outcome = client_->GetObject(object_request);
// if (get_object_outcome.IsSuccess()) {
// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
// local_file << get_object_outcome.GetResult().GetBody().rdbuf();
// return Status::OK();
// } else {
// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
// << get_object_outcome.GetError().GetMessage() << std::endl;
// return Status::Error(get_object_outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
// Aws::S3::Model::DeleteObjectRequest object_request;
// object_request.WithBucket(bucket_name).WithKey(object_key);
//
// auto delete_object_outcome = client_->DeleteObject(object_request);
//
// if (delete_object_outcome.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "DeleteObject error: " <<
// delete_object_outcome.GetError().GetExceptionName() << " " <<
// delete_object_outcome.GetError().GetMessage() << std::endl;
//
// return Status::Error(delete_object_outcome.GetError().GetMessage());
// }
//}
//
//}
//}
//}
//}
\ No newline at end of file
cpp/src/storage/s3/S3ClientWrapper.h 已删除 100644 → 0
浏览文件 @ 80dbaebd
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//#pragma once
//
//#include "storage/IStorage.h"
//
//
//#include <aws/s3/S3Client.h>
//#include <aws/core/Aws.h>
//#include <aws/core/auth/AWSCredentialsProvider.h>
//
//
//using namespace Aws::S3;
//using namespace Aws::S3::Model;
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//class S3ClientWrapper : public IStorage {
// public:
//
// S3ClientWrapper() = default;
// ~S3ClientWrapper() = default;
//
// Status Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) override;
// Status Close() override;
//
// Status CreateBucket(std::string& bucket_name) override;
// Status DeleteBucket(std::string& bucket_name) override;
// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
//
// private:
// S3Client *client_ = nullptr;
// Aws::SDKOptions options_;
//};
//
//}
//}
//}
//}
\ No newline at end of file
cpp/thirdparty/versions.txt
浏览文件 @ 8464b8a2
ARROW_VERSION=zilliz
BOOST_VERSION=1.70.0 BOOST_VERSION=1.70.0
BZIP2_VERSION=1.0.6 BZIP2_VERSION=1.0.6
EASYLOGGINGPP_VERSION=v9.96.7 EASYLOGGINGPP_VERSION=v9.96.7
FAISS_VERSION=branch-0.1.0
MKL_VERSION=2019.4.243
GTEST_VERSION=1.8.1 GTEST_VERSION=1.8.1
JSONCONS_VERSION=0.126.0
LAPACK_VERSION=v3.8.0
LZ4_VERSION=v1.9.1 LZ4_VERSION=v1.9.1
MYSQLPP_VERSION=3.2.4 MYSQLPP_VERSION=3.2.4
OPENBLAS_VERSION=v0.3.6
PROMETHEUS_VERSION=v0.7.0 PROMETHEUS_VERSION=v0.7.0
ROCKSDB_VERSION=v6.0.2
SNAPPY_VERSION=1.1.7 SNAPPY_VERSION=1.1.7
SQLITE_VERSION=3280000 SQLITE_VERSION=3280000
SQLITE_ORM_VERSION=master SQLITE_ORM_VERSION=master
YAMLCPP_VERSION=0.6.2 YAMLCPP_VERSION=0.6.2
ZLIB_VERSION=v1.2.11 ZLIB_VERSION=v1.2.11
ZSTD_VERSION=v1.4.0 ZSTD_VERSION=v1.4.0
AWS_VERSION=1.7.125
LIBUNWIND_VERSION=1.3.1 LIBUNWIND_VERSION=1.3.1
GPERFTOOLS_VERSION=2.7 GPERFTOOLS_VERSION=2.7
GRPC_VERSION=master GRPC_VERSION=master
......
cpp/unittest/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,20 +17,70 @@ ...@@ -17,20 +17,70 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
link_directories( include_directories(${MILVUS_SOURCE_DIR})
"${CMAKE_BINARY_DIR}/lib" include_directories(${MILVUS_ENGINE_SRC})
)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(grpc_service_files
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_root_files)
set(scheduler_files
${scheduler_action_files}
${scheduler_event_files}
${scheduler_job_files}
${scheduler_resource_files}
${scheduler_task_files}
${scheduler_root_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
set(unittest_srcs set(unittest_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
set(require_files set(helper_files
${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
) )
set(unittest_libs set(unittest_libs
...@@ -49,15 +99,16 @@ set(unittest_libs ...@@ -49,15 +99,16 @@ set(unittest_libs
z z
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart cudart
cublas
) )
foreach(dir ${CORE_INCLUDE_DIRS}) foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir}) include_directories(${dir})
endforeach() endforeach()
add_subdirectory(server)
add_subdirectory(db) add_subdirectory(db)
add_subdirectory(knowhere) add_subdirectory(knowhere)
add_subdirectory(metrics) add_subdirectory(metrics)
add_subdirectory(scheduler) add_subdirectory(scheduler)
add_subdirectory(server)
#add_subdirectory(storage) #add_subdirectory(storage)
\ No newline at end of file
cpp/unittest/db/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,61 +17,27 @@ ...@@ -17,61 +17,27 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) set(db_test_files
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) ${cache_files}
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) ${config_files}
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) ${db_main_files}
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) ${db_engine_files}
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) ${db_insert_files}
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) ${db_meta_files}
${db_scheduler_files}
aux_source_directory(./ test_srcs) ${metrics_files}
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_srcs
${scheduler_files} ${scheduler_files}
${scheduler_context_files} ${knowhere_files}
${scheduler_task_files} ${helper_files}
) ${test_files}
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
set(db_test_src
${config_files}
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
${knowhere_src}
${util_files}
${require_files}
${test_srcs}
) )
cuda_add_executable(db_test ${db_test_src}) cuda_add_executable(db_test ${db_test_files})
set(db_libs set(db_libs
sqlite sqlite
...@@ -81,13 +47,7 @@ set(db_libs ...@@ -81,13 +47,7 @@ set(db_libs
mysqlpp mysqlpp
) )
set(knowhere_libs target_link_libraries(db_test ${db_libs} knowhere ${unittest_libs})
knowhere
cudart
cublas
)
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
install(TARGETS db_test DESTINATION unittest) install(TARGETS db_test DESTINATION unittest)
cpp/unittest/knowhere/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -20,15 +20,15 @@ ...@@ -20,15 +20,15 @@
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(knowhere_src set(knowhere_files
${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp
${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp
${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp)
set(helper set(helper_files
utils.cpp utils.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h) )
set(knowhere_libs set(knowhere_libs
knowhere knowhere
...@@ -36,7 +36,7 @@ set(knowhere_libs ...@@ -36,7 +36,7 @@ set(knowhere_libs
cublas cublas
) )
add_executable(knowhere_test knowhere_test.cpp ${knowhere_src} ${helper}) add_executable(knowhere_test knowhere_test.cpp ${knowhere_files} ${helper_files})
target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs}) target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs})
install(TARGETS knowhere_test DESTINATION unittest) install(TARGETS knowhere_test DESTINATION unittest)
\ No newline at end of file
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,86 +17,35 @@ ...@@ -17,86 +17,35 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
include_directories(../../src) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
include_directories(/usr/include) include_directories(/usr/include)
include_directories(../../third_party/build/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories(../../third_party/build/lib) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(../../src/metrics)
include_directories(/usr/include/mysql) set(metrics_test_files
${cache_files}
set(count_test_src
${config_files} ${config_files}
${cache_srcs}
${db_main_files} ${db_main_files}
${db_engine_files} ${db_engine_files}
${db_insert_files} ${db_insert_files}
${db_meta_files} ${db_meta_files}
${db_scheduler_srcs} ${db_scheduler_files}
${wrapper_src} ${metrics_files}
${scheduler_action_srcs} ${scheduler_files}
${scheduler_event_srcs} ${knowhere_files}
${scheduler_resource_srcs} ${helper_files}
${scheduler_task_srcs} ${test_files}
${scheduler_srcs}
${knowhere_src}
${metrics_src}
${test_srcs}
${util_files}
) )
add_executable(metrics_test ${count_test_src} ${require_files} ) add_executable(metrics_test ${metrics_test_files})
set(knowhere_libs
knowhere
)
target_link_libraries(metrics_test target_link_libraries(metrics_test
${knowhere_libs} knowhere
cudart
cublas
sqlite sqlite
boost_system_static boost_system_static
boost_filesystem_static boost_filesystem_static
lz4 lz4
metrics
gtest
pthread
z
mysqlpp mysqlpp
${unittest_libs} ${unittest_libs}
) )
......
cpp/unittest/scheduler/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,62 +17,27 @@ ...@@ -17,62 +17,27 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files})
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
set(scheduler_test_src
${unittest_srcs}
${test_srcs}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
set(scheduler_test_files
${cache_files}
${config_files} ${config_files}
${cache_srcs}
${db_main_files} ${db_main_files}
${db_engine_files} ${db_engine_files}
${db_insert_files} ${db_insert_files}
${db_meta_files} ${db_meta_files}
${db_scheduler_srcs} ${db_scheduler_files}
${wrapper_src} ${scheduler_files}
${knowhere_src} ${knowhere_files}
${util_files} ${helper_files}
${require_files} ${unittest_files}
${test_files}
) )
cuda_add_executable(scheduler_test ${scheduler_test_src}) cuda_add_executable(scheduler_test ${scheduler_test_files})
set(scheduler_libs set(scheduler_libs
sqlite sqlite
...@@ -82,14 +47,7 @@ set(scheduler_libs ...@@ -82,14 +47,7 @@ set(scheduler_libs
mysqlpp mysqlpp
) )
set(knowhere_libs target_link_libraries(scheduler_test ${scheduler_libs} knowhere ${unittest_libs})
knowhere
cudart
cublas
)
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})
install(TARGETS scheduler_test DESTINATION unittest) install(TARGETS scheduler_test DESTINATION unittest)
cpp/unittest/server/CMakeLists.txt
浏览文件 @ 8464b8a2
...@@ -17,75 +17,19 @@ ...@@ -17,75 +17,19 @@
# under the License. # under the License.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
include_directories(${MILVUS_ENGINE_SRC}/) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
include_directories(/usr/include)
include_directories(/usr/include/mysql)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
aux_source_directory(./ test_srcs)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(util_files include_directories(/usr/include)
${MILVUS_ENGINE_SRC}/utils/Status.cpp include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
set(db_src include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
${config_files} include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
${knowhere_src}
${util_files}
${require_files}
${test_srcs}
)
set(utils_srcs set(helper_files ${helper_files}
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp)
${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp
)
set(grpc_service_files set(grpc_service_files
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
...@@ -94,44 +38,49 @@ set(grpc_service_files ...@@ -94,44 +38,49 @@ set(grpc_service_files
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
) )
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) set(server_test_files
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) ${cache_files}
${config_files}
cuda_add_executable(server_test ${db_main_files}
${db_src} ${db_engine_files}
${unittest_srcs} ${db_insert_files}
${grpc_server_src} ${db_meta_files}
${server_src} ${db_scheduler_files}
${utils_srcs} ${scheduler_files}
${server_files}
${grpc_server_files}
${grpc_service_files} ${grpc_service_files}
${require_files} ${knowhere_files}
${helper_files}
${unittest_files}
${test_files}
) )
cuda_add_executable(server_test ${server_test_files})
set(client_grpc_lib
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc)
set(require_libs set(require_libs
knowhere knowhere
stdc++ stdc++
cudart
cublas
sqlite sqlite
mysqlpp mysqlpp
boost_system_static boost_system_static
boost_filesystem_static boost_filesystem_static
snappy snappy
z
bz2 bz2
zstd zstd
lz4 lz4
pthread ${client_grpc_lib}
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc
) )
target_link_libraries(server_test target_link_libraries(server_test
${require_libs} ${require_libs}
${cuda_library}
${unittest_libs} ${unittest_libs}
) )
......
cpp/unittest/storage/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ 80dbaebd
#-------------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(s3_client_test_src
${unittest_srcs}
${s3_client_src}
${require_files}
S3ClientTest.cpp
${MILVUS_ENGINE_SRC}/db/Status.cpp
)
add_executable(s3_test
${s3_client_test_src}
${config_files}
${util_files}
)
set(s3_client_libs
stdc++
aws-cpp-sdk-s3
aws-cpp-sdk-core
boost_filesystem_static
)
target_link_libraries(s3_test
${s3_client_libs}
${unittest_libs}
curl
crypto)
install(TARGETS s3_test DESTINATION unittest)
\ No newline at end of file
cpp/unittest/storage/S3ClientTest.cpp 已删除 100644 → 0
浏览文件 @ 80dbaebd
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
//#include "storage/IStorage.h"
//#include "storage/s3/S3ClientWrapper.h"
//#include <gtest/gtest.h>
//#include <memory.h>
//#include <fstream>
//
//
//using namespace zilliz::milvus::engine;
//
//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
//
// std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
//
// std::string ip_address = "127.0.0.1";
// std::string port = "9000";
// std::string access_key = "AKIAIOSFODNN7EXAMPLE";
// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
//
// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
// ASSERT_TRUE(status.ok());
//
// std::string filename = "/tmp/s3_test_file";
// std::string bucket_name = "bucktname";
// std::string object_name = "test_file";
//
// status = storage_ptr->CreateBucket(bucket_name);
// std::cout << status.IsAlreadyExist() << std::endl;
// if (status.IsAlreadyExist()) {
// status = storage_ptr->DeleteBucket(bucket_name);
// status = storage_ptr->CreateBucket(bucket_name);
// }
//
// ASSERT_TRUE(status.ok());
//
// std::ofstream ofile(filename);
// std::stringstream ss;
// for (int i = 0; i < 1024; ++i) {
// ss << i;
// }
// ofile << ss.str() << std::endl;
// ofile.close();
// status = storage_ptr->UploadFile(bucket_name, object_name, filename);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
// std::ifstream infile(filename);
// std::string in_buffer;
// infile >> in_buffer;
// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
//
// status = storage_ptr->DeleteFile(bucket_name, object_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DeleteBucket(bucket_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->Close();
// ASSERT_TRUE(status.ok());
//}
//
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或