From 8464b8a206220abe0739805ad01975dae7f8b2e1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 18 Sep 2019 15:49:47 +0800 Subject: [PATCH] refactor cmake Former-commit-id: 587cc22f501fd12cbdaf2d9950fce3f2483c7020 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 31 +- cpp/cmake/DefineOptions.cmake | 22 +- cpp/cmake/ThirdPartyPackages.cmake | 801 +----------------- cpp/src/CMakeLists.txt | 116 ++- .../sdk/examples/grpcsimple/CMakeLists.txt | 5 - cpp/src/storage/IStorage.h | 50 -- cpp/src/storage/s3/S3ClientWrapper.cpp | 177 ---- cpp/src/storage/s3/S3ClientWrapper.h | 63 -- cpp/thirdparty/versions.txt | 8 - cpp/unittest/CMakeLists.txt | 63 +- cpp/unittest/db/CMakeLists.txt | 78 +- cpp/unittest/knowhere/CMakeLists.txt | 8 +- cpp/unittest/metrics/CMakeLists.txt | 77 +- cpp/unittest/scheduler/CMakeLists.txt | 68 +- cpp/unittest/server/CMakeLists.txt | 117 +-- cpp/unittest/storage/CMakeLists.txt | 55 -- cpp/unittest/storage/S3ClientTest.cpp | 78 -- 18 files changed, 225 insertions(+), 1593 deletions(-) delete mode 100644 cpp/src/storage/IStorage.h delete mode 100644 cpp/src/storage/s3/S3ClientWrapper.cpp delete mode 100644 cpp/src/storage/s3/S3ClientWrapper.h delete mode 100644 cpp/unittest/storage/CMakeLists.txt delete mode 100644 cpp/unittest/storage/S3ClientTest.cpp diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c5d27e34..6d665555 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-556 - Add Job Definition in Scheduler - MS-558 - Refine status code - MS-562 - Add JobMgr and TaskCreator in Scheduler +- MS-566 - Refactor cmake ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 69aaa9f0..91c7b528 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -51,13 +51,6 @@ message(STATUS "Build type = ${BUILD_TYPE}") project(milvus VERSION "${MILVUS_VERSION}") project(milvus_engine LANGUAGES CUDA CXX) -# Ensure that a default make is set -if("${MAKE}" STREQUAL "") - if(NOT MSVC) - find_program(MAKE make) - endif() -endif() - set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}") set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}") set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}") @@ -76,9 +69,6 @@ message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " "(full: '${MILVUS_VERSION}')") -set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) -set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) - find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") @@ -105,8 +95,28 @@ else() set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() +# Ensure that a default make is set +if("${MAKE}" STREQUAL "") + if(NOT MSVC) + find_program(MAKE make) + endif() +endif() + +find_path(MYSQL_INCLUDE_DIR + NAMES "mysql.h" + PATH_SUFFIXES "mysql") +if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND") + message(FATAL_ERROR "Could not found MySQL include directory") +else() + include_directories(${MYSQL_INCLUDE_DIR}) +endif() + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) + include(ExternalProject) include(DefineOptions) include(BuildUtils) @@ -114,7 +124,6 @@ include(ThirdPartyPackages) config_summary() -set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_subdirectory(src) if (BUILD_UNIT_TEST STREQUAL "ON") diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 1484a9d2..2b9a68e8 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) -define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF) - define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ Note that this requires linking Boost statically" ON) @@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON) define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) -define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF) - -define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF) - -define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF) - define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON) -define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF) - -define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF) - define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) -define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF) - define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON) define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON) @@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) -define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF) - -#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON) - if(CMAKE_VERSION VERSION_LESS 3.7) set(MILVUS_WITH_ZSTD_DEFAULT OFF) else() @@ -106,13 +88,13 @@ else() endif() define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT}) -define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON) - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) endif() +define_option(MILVUS_WITH_GRPC "Build with GRPC" ON) + #---------------------------------------------------------------------- if(MSVC) set_option_category("MSVC") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fa61da18..f280c482 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -16,27 +16,18 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES - ARROW BOOST BZip2 - FAISS GTest - Knowhere - JSONCONS - LAPACK Lz4 MySQLPP - OpenBLAS Prometheus - RocksDB Snappy SQLite SQLite_ORM - Thrift yaml-cpp ZLIB ZSTD - AWS libunwind gperftools GRPC) @@ -51,30 +42,16 @@ foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) endforeach() macro(build_dependency DEPENDENCY_NAME) - if("${DEPENDENCY_NAME}" STREQUAL "ARROW") - build_arrow() - elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") + if("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() - elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") - build_faiss() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") build_gtest() - elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") - build_lapack() - elseif("${DEPENDENCY_NAME}" STREQUAL "Knowhere") - build_knowhere() elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") build_lz4() elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP") build_mysqlpp() - elseif ("${DEPENDENCY_NAME}" STREQUAL "JSONCONS") - build_jsoncons() - elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") - build_openblas() elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") build_prometheus() - elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB") - build_rocksdb() elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") build_snappy() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") @@ -87,8 +64,6 @@ macro(build_dependency DEPENDENCY_NAME) build_zlib() elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") build_zstd() - elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") - build_aws() elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") build_libunwind() elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") @@ -280,8 +255,7 @@ if(DEFINED ENV{MILVUS_BOOST_URL}) else() string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) set(BOOST_SOURCE_URL - "http://192.168.1.201/artifactory/generic-local/tools/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz") -# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" + "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz") endif() set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26") @@ -292,22 +266,6 @@ else() endif() set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") -if(DEFINED ENV{MILVUS_FAISS_URL}) - set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}") -else() - set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") - # set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") -endif() - -# set(FAISS_MD5 "a589663865a8558205533c8ac414278c") -set(FAISS_MD5 "31167ecbd1903fec600dc4ac00b9be9e") - -if(DEFINED ENV{MILVUS_KNOWHERE_URL}) - set(KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}") -else() - set(KNOWHERE_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/knowhere") -endif() - if (DEFINED ENV{MILVUS_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}") else () @@ -316,20 +274,6 @@ else () endif() set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") -if (DEFINED ENV{MILVUS_JSONCONS_URL}) - set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}") -else () - set(JSONCONS_SOURCE_URL - "https://github.com/danielaparker/jsoncons/archive/v${JSONCONS_VERSION}.tar.gz") -endif() - -if(DEFINED ENV{MILVUS_LAPACK_URL}) - set(LAPACK_SOURCE_URL "$ENV{MILVUS_LAPACK_URL}") -else() - set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") -endif() -set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2") - if(DEFINED ENV{MILVUS_LZ4_URL}) set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}") else() @@ -344,14 +288,6 @@ else() endif() set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") -if (DEFINED ENV{MILVUS_OPENBLAS_URL}) - set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}") -else () - set(OPENBLAS_SOURCE_URL - "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") -endif() -set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") - if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") else () @@ -359,15 +295,6 @@ else () https://github.com/jupp0r/prometheus-cpp.git) endif() - -if (DEFINED ENV{MILVUS_ROCKSDB_URL}) - set(ROCKSDB_SOURCE_URL "$ENV{MILVUS_ROCKSDB_URL}") -else () - set(ROCKSDB_SOURCE_URL - "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz") -endif() -set(ROCKSDB_MD5 "a8f2f594182e97a08629bcc66dfd3fa0") - if(DEFINED ENV{MILVUS_SNAPPY_URL}) set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}") else() @@ -413,13 +340,6 @@ else() endif() set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120") -if(DEFINED ENV{MILVUS_AWS_URL}) - set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}") -else() - set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") -endif() -set(AWS_MD5 "9217f5bc8bf23dea04f4466521c85fd9") - if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") else() @@ -445,120 +365,6 @@ endif() set(GRPC_MD5 "7ec59ad54c85a12dcbbfede09bf413a9") -# ---------------------------------------------------------------------- -# ARROW - -macro(build_arrow) - message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") - set(ARROW_STATIC_LIB_NAME arrow) - - set(ARROW_STATIC_LIB - "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") - set(ARROW_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DARROW_BUILD_STATIC=ON - -DARROW_BUILD_SHARED=OFF - -DARROW_PARQUET=ON - -DARROW_USE_GLOG=OFF - -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX} - "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs" - -DCMAKE_BUILD_TYPE=Release) - - if(USE_JFROG_CACHE STREQUAL "ON") - execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) - if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") - string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}") - set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz") - set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}") - set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${ARROW_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) - else() - file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!") - endif() - else() - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - endif() - - file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") - add_library(arrow STATIC IMPORTED) - set_target_properties(arrow - PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - - add_dependencies(arrow arrow_ep) - - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") - - add_custom_command(TARGET arrow_ep POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ - COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/ - DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a) - -endmacro() - -if(MILVUS_WITH_ARROW) - - resolve_dependency(ARROW) - - link_directories(SYSTEM ${ARROW_PREFIX}/lib/) - include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) -endif() - # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu (incubating)) @@ -755,370 +561,6 @@ if(MILVUS_WITH_BZ2) include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") endif() -# ---------------------------------------------------------------------- -# Knowhere - -macro(build_knowhere) - message(STATUS "Building knowhere from source") - set(KNOWHERE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/knowhere_ep-prefix/src/knowhere_ep") - set(KNOWHERE_INCLUDE_DIR "${KNOWHERE_PREFIX}/include") - set(KNOWHERE_STATIC_LIB - "${KNOWHERE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}knowhere${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(KNOWHERE_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${KNOWHERE_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib - "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}" - "-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}" - -DCMAKE_BUILD_TYPE=Release) - - externalproject_add(knowhere_ep - URL - ${KNOWHERE_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${KNOWHERE_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${KNOWHERE_STATIC_LIB}) - - file(MAKE_DIRECTORY "${KNOWHERE_INCLUDE_DIR}") - add_library(knowhere STATIC IMPORTED) - set_target_properties( - knowhere - PROPERTIES IMPORTED_LOCATION "${KNOWHERE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${KNOWHERE_INCLUDE_DIR}") - - add_dependencies(knowhere knowhere_ep) -endmacro() - -if(MILVUS_WITH_KNOWHERE) - resolve_dependency(Knowhere) - - get_target_property(KNOWHERE_INCLUDE_DIR knowhere INTERFACE_INCLUDE_DIRECTORIES) - link_directories(SYSTEM "${KNOWHERE_PREFIX}/lib") - include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}") - include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService") -endif() - -# ---------------------------------------------------------------------- -# OpenBLAS - -macro(build_openblas) - message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") - set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") - set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") - set(OPENBLAS_STATIC_LIB - "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") - - if(USE_JFROG_CACHE STREQUAL "ON") - set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz") - set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}") - set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${OPENBLAS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${OPENBLAS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(openblas_ep - URL - ${OPENBLAS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} - PREFIX=${OPENBLAS_PREFIX} - install - BUILD_BYPRODUCTS - ${OPENBLAS_STATIC_LIB}) - - ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) - else() - file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(openblas_ep - URL - ${OPENBLAS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} - PREFIX=${OPENBLAS_PREFIX} - install - BUILD_BYPRODUCTS - ${OPENBLAS_STATIC_LIB}) - endif() - - file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") - add_library(openblas STATIC IMPORTED) - set_target_properties( - openblas - PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}") - - add_dependencies(openblas openblas_ep) -endmacro() - -# ---------------------------------------------------------------------- -# LAPACK - -macro(build_lapack) - message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") - set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") - set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") - set(LAPACK_STATIC_LIB - "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(LAPACK_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib) - - if(USE_JFROG_CACHE STREQUAL "ON") - set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz") - set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}") - set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${LAPACK_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${LAPACK_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(lapack_ep - URL - ${LAPACK_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${LAPACK_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${LAPACK_STATIC_LIB}) - - ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) - else() - file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(lapack_ep - URL - ${LAPACK_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${LAPACK_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${LAPACK_STATIC_LIB}) - endif() - - file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") - add_library(lapack STATIC IMPORTED) - set_target_properties( - lapack - PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}") - - add_dependencies(lapack lapack_ep) -endmacro() - -# ---------------------------------------------------------------------- -# FAISS - -if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) -endif() - -if(EXISTS "/proc/cpuinfo") - FILE(READ /proc/cpuinfo PROC_CPUINFO) - - SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") - STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") - STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") - - if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel") - set(BUILD_FAISS_WITH_MKL OFF) - endif() -endif() - -macro(build_faiss) - message(STATUS "Building FAISS-${FAISS_VERSION} from source") - set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") - set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") - set(FAISS_STATIC_LIB - "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(FAISS_CONFIGURE_ARGS - "--prefix=${FAISS_PREFIX}" - "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS}" - --without-python) - - set(FAISS_CFLAGS ${EP_C_FLAGS}) - set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - message(STATUS "Build Faiss with MKL") - if(NOT DEFINED MKL_LIB_PATH) - set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") - message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") - endif() - - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" - "LDFLAGS=-L${MKL_LIB_PATH}" - "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") - - else() - message(STATUS "Build Faiss with OpenBlas/LAPACK") - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") - endif() - - if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" - "--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_75,code=sm_75" - ) - else() - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) - endif() - - if(USE_JFROG_CACHE STREQUAL "ON") - string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") - set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") - set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - endif() - - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) - else() - file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - endif() - endif() - - file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") - add_library(faiss SHARED IMPORTED) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a - ${MKL_LIB_PATH}/libmkl_gnu_thread.a - ${MKL_LIB_PATH}/libmkl_core.a) - - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" ) - else() - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) - endif() - - add_dependencies(faiss faiss_ep) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - add_dependencies(faiss openblas_ep) - add_dependencies(faiss lapack_ep) - endif() - -endmacro() - -if(MILVUS_WITH_FAISS) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") - endif() - - resolve_dependency(FAISS) - get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") - link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -endif() - # ---------------------------------------------------------------------- # Google gtest @@ -1232,30 +674,6 @@ if (MILVUS_BUILD_TESTS) include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# JSONCONS - -macro(build_jsoncons) - message(STATUS "Building JSONCONS-${JSONCONS_VERSION} from source") - - set(JSONCONS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jsoncons_ep-prefix") - set(JSONCONS_TAR_NAME "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}.tar.gz") - set(JSONCONS_INCLUDE_DIR "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}/include") - if (NOT EXISTS ${JSONCONS_INCLUDE_DIR}) - file(MAKE_DIRECTORY ${JSONCONS_PREFIX}) - file(DOWNLOAD ${JSONCONS_SOURCE_URL} - ${JSONCONS_TAR_NAME}) - execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${JSONCONS_TAR_NAME} - WORKING_DIRECTORY ${JSONCONS_PREFIX}) - - endif () -endmacro() - -if(MILVUS_WITH_JSONCONS) - resolve_dependency(JSONCONS) - include_directories(SYSTEM "${JSONCONS_INCLUDE_DIR}") -endif() - # ---------------------------------------------------------------------- # lz4 @@ -1563,95 +981,6 @@ if(MILVUS_WITH_PROMETHEUS) endif() -# ---------------------------------------------------------------------- -# RocksDB - -macro(build_rocksdb) - message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source") - set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep") - set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include") - set(ROCKSDB_STATIC_LIB_NAME rocksdb) - set(ROCKSDB_STATIC_LIB - "${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - if(USE_JFROG_CACHE STREQUAL "ON") - string(MD5 ROCKSDB_COMBINE_MD5 "${ROCKSDB_LAST_COMMIT_ID}") - set(ROCKSDB_CACHE_PACKAGE_NAME "rocksdb_${ROCKSDB_MD5}.tar.gz") - set(ROCKSDB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ROCKSDB_CACHE_PACKAGE_NAME}") - set(ROCKSDB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ROCKSDB_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ROCKSDB_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${ROCKSDB_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(rocksdb_ep - URL - ${ROCKSDB_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - static_lib - "prefix=${ROCKSDB_PREFIX}" - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - install-static - "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" - BUILD_BYPRODUCTS - "${ROCKSDB_STATIC_LIB}") - - ExternalProject_Create_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ROCKSDB_CACHE_URL}) - else() - file(DOWNLOAD ${ROCKSDB_CACHE_URL} ${ROCKSDB_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ROCKSDB_CACHE_URL} TO ${ROCKSDB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(rocksdb_ep - URL - ${ROCKSDB_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - static_lib - "prefix=${ROCKSDB_PREFIX}" - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - install-static - "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" - BUILD_BYPRODUCTS - "${ROCKSDB_STATIC_LIB}") - endif() - - file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include") - - add_library(rocksdb STATIC IMPORTED) - set_target_properties(rocksdb - PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}") - add_dependencies(rocksdb rocksdb_ep) -endmacro() - -if(MILVUS_WITH_ROCKSDB) - - resolve_dependency(RocksDB) - - link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) - include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) -endif() - # ---------------------------------------------------------------------- # Snappy @@ -2091,116 +1420,6 @@ if(MILVUS_WITH_ZSTD) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# aws -macro(build_aws) - message(STATUS "Building aws-${AWS_VERSION} from source") - set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep") - - set(AWS_CMAKE_ARGS - ${EP_COMMON_TOOLCHAIN} - "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3 - -DBUILD_SHARED_LIBS=off - -DENABLE_TESTING=off - -DENABLE_UNITY_BUILD=on - -DNO_ENCRYPTION=off) - - set(AWS_CPP_SDK_CORE_STATIC_LIB - "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWS_CPP_SDK_S3_STATIC_LIB - "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include") - set(AWS_CMAKE_ARGS - ${AWS_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - - if(USE_JFROG_CACHE STREQUAL "ON") - set(AWS_CACHE_PACKAGE_NAME "aws_${AWS_MD5}.tar.gz") - set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}") - set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${AWS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${AWS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(aws_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${AWS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${AWS_PREFIX} - URL - ${AWS_SOURCE_URL} - BUILD_BYPRODUCTS - "${AWS_CPP_SDK_S3_STATIC_LIB}" - "${AWS_CPP_SDK_CORE_STATIC_LIB}") - - ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL}) - else() - file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(aws_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${AWS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${AWS_PREFIX} - URL - ${AWS_SOURCE_URL} - BUILD_BYPRODUCTS - "${AWS_CPP_SDK_S3_STATIC_LIB}" - "${AWS_CPP_SDK_CORE_STATIC_LIB}") - endif() - - file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}") - add_library(aws-cpp-sdk-s3 STATIC IMPORTED) - set_target_properties(aws-cpp-sdk-s3 - PROPERTIES - IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") - - add_library(aws-cpp-sdk-core STATIC IMPORTED) - set_target_properties(aws-cpp-sdk-core - PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") - - add_dependencies(aws-cpp-sdk-s3 aws_ep) - add_dependencies(aws-cpp-sdk-core aws_ep) - -endmacro() - -if(MILVUS_WITH_AWS) - resolve_dependency(AWS) - - link_directories(SYSTEM ${AWS_PREFIX}/lib) - - get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM ${AWS_CPP_SDK_S3_INCLUDE_DIR}) - - get_target_property(AWS_CPP_SDK_CORE_INCLUDE_DIR aws-cpp-sdk-core INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR}) - -endif() - # ---------------------------------------------------------------------- # libunwind @@ -2465,12 +1684,14 @@ macro(build_grpc) add_dependencies(grpc_protoc grpc_ep) endmacro() -resolve_dependency(GRPC) +if(MILVUS_WITH_GRPC) + resolve_dependency(GRPC) -get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) -include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) -link_directories(SYSTEM ${GRPC_PREFIX}/lib) + get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) + link_directories(SYSTEM ${GRPC_PREFIX}/lib) -set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) -include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) -link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) + set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) + include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) + link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) +endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index c8cba4d9..b7e71966 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -17,25 +17,41 @@ # under the License. #------------------------------------------------------------------------------- +include_directories(${MILVUS_SOURCE_DIR}) +include_directories(${MILVUS_ENGINE_SRC}) add_subdirectory(core) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) -foreach(dir ${CORE_INCLUDE_DIRS}) +foreach (dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) -endforeach() +endforeach () aux_source_directory(cache cache_files) + aux_source_directory(config config_files) -aux_source_directory(server server_files) -aux_source_directory(server/grpc_impl grpcserver_files) -aux_source_directory(utils utils_files) + aux_source_directory(db db_main_files) aux_source_directory(db/engine db_engine_files) aux_source_directory(db/insert db_insert_files) aux_source_directory(db/meta db_meta_files) +aux_source_directory(db/scheduler scheduler_files) +aux_source_directory(db/scheduler/context scheduler_context_files) +aux_source_directory(db/scheduler/task scheduler_task_files) +set(db_scheduler_files + ${scheduler_files} + ${scheduler_context_files} + ${scheduler_task_files} + ) + +set(grpc_service_files + grpc/gen-milvus/milvus.grpc.pb.cc + grpc/gen-milvus/milvus.pb.cc + grpc/gen-status/status.grpc.pb.cc + grpc/gen-status/status.pb.cc + ) + aux_source_directory(metrics metrics_files) -aux_source_directory(wrapper/knowhere knowhere_files) aux_source_directory(scheduler/action scheduler_action_files) aux_source_directory(scheduler/event scheduler_event_files) @@ -43,7 +59,7 @@ aux_source_directory(scheduler/job scheduler_job_files) aux_source_directory(scheduler/resource scheduler_resource_files) aux_source_directory(scheduler/task scheduler_task_files) aux_source_directory(scheduler scheduler_root_files) -set(scheduler_srcs +set(scheduler_files ${scheduler_action_files} ${scheduler_event_files} ${scheduler_job_files} @@ -52,32 +68,12 @@ set(scheduler_srcs ${scheduler_root_files} ) -aux_source_directory(db/scheduler scheduler_files) -aux_source_directory(db/scheduler/context scheduler_context_files) -aux_source_directory(db/scheduler/task scheduler_task_files) -set(db_scheduler_files - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) -set(license_check_files - license/LicenseLibrary.cpp - license/LicenseCheck.cpp - ) +aux_source_directory(server server_files) +aux_source_directory(server/grpc_impl grpc_server_files) -set(license_generator_files - license/LicenseGenerator.cpp - license/LicenseLibrary.cpp - ) +aux_source_directory(utils utils_files) -set(grpc_service_files - metrics/SystemInfo.cpp - metrics/SystemInfo.h - grpc/gen-milvus/milvus.grpc.pb.cc - grpc/gen-milvus/milvus.pb.cc - grpc/gen-status/status.grpc.pb.cc - grpc/gen-status/status.pb.cc - scheduler/Utils.h) +aux_source_directory(wrapper/knowhere knowhere_files) set(db_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp @@ -88,17 +84,12 @@ set(db_files ${db_meta_files} ${db_scheduler_files} ${metrics_files} - ${knowhere_files} ${utils_files} + ${knowhere_files} ) -set(s3_client_files - storage/s3/S3ClientWrapper.cpp - storage/s3/S3ClientWrapper.h) - include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -include_directories(/usr/include/mysql) include_directories(grpc/gen-status) include_directories(grpc/gen-milvus) @@ -109,17 +100,22 @@ set(client_grpc_lib grpc_protobuf grpc_protoc) -set(third_party_libs - knowhere - sqlite - ${client_grpc_lib} - yaml-cpp +set(prometheus_lib prometheus-cpp-push prometheus-cpp-pull - prometheus-cpp-core + prometheus-cpp-core) + +set(boost_lib boost_system_static boost_filesystem_static - boost_serialization_static + boost_serialization_static) + +set(third_party_libs + sqlite + ${client_grpc_lib} + yaml-cpp + ${prometheus_lib} + ${boost_lib} bzip2 lz4 snappy @@ -131,13 +127,12 @@ set(third_party_libs ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so cudart ) - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} gperftools - libunwind) -endif() - + libunwind + ) +endif () link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(engine_libs @@ -149,12 +144,11 @@ set(engine_libs if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(engine_libs - ${engine_libs} - libquadmath.a - ) + ${engine_libs} + libquadmath.a + ) endif () - cuda_add_library(milvus_engine STATIC ${db_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) @@ -162,9 +156,7 @@ add_library(metrics STATIC ${metrics_files}) set(metrics_lib yaml-cpp - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core + ${prometheus_lib} ) target_link_libraries(metrics ${metrics_lib}) @@ -176,21 +168,17 @@ set(server_libs metrics ) -set(knowhere_libs - knowhere - ) - add_executable(milvus_server ${config_files} + ${metrics_files} + ${scheduler_files} ${server_files} - ${grpcserver_files} - ${utils_files} + ${grpc_server_files} ${grpc_service_files} - ${metrics_files} - ${scheduler_srcs} + ${utils_files} ) -target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs}) +target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs}) install(TARGETS milvus_server DESTINATION bin) diff --git a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt index b6c67061..77542ed2 100644 --- a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt +++ b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt @@ -20,11 +20,6 @@ aux_source_directory(src src_files) -include_directories(src) -include_directories(../../include) - -link_directories(${CMAKE_BINARY_DIR}) - add_executable(sdk_simple main.cpp ${src_files} diff --git a/cpp/src/storage/IStorage.h b/cpp/src/storage/IStorage.h deleted file mode 100644 index 431dd45c..00000000 --- a/cpp/src/storage/IStorage.h +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include "db/Status.h" - -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -class IStorage { - public: - virtual Status Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) = 0; - - virtual Status Close() = 0; - - virtual Status CreateBucket(std::string& bucket_name) = 0; - virtual Status DeleteBucket(std::string& bucket_name) = 0; - virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; - virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; - virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0; -}; - -} -} -} -} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp deleted file mode 100644 index 2dca5b3e..00000000 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -//#include "S3ClientWrapper.h" -// -//#include -//#include -//#include -//#include -//#include -// -//#include -//#include -// -// -//namespace zilliz { -//namespace milvus { -//namespace engine { -//namespace storage { -// -//Status -//S3ClientWrapper::Create(const std::string &ip_address, -// const std::string &port, -// const std::string &access_key, -// const std::string &secret_key) { -// Aws::InitAPI(options_); -// Aws::Client::ClientConfiguration cfg; -// -// // TODO: ip_address need to be validated. -// -// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port -// cfg.scheme = Aws::Http::Scheme::HTTP; -// cfg.verifySSL = -// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key -// client_ = -// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), -// cfg, -// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, -// false); -// if (client_ == nullptr) { -// std::string error = "Can't connect server."; -// return Status::Error(error); -// } else { -// return Status::OK(); -// } -//} -// -// -//Status -//S3ClientWrapper::Close() { -// if (client_ != nullptr) { -// delete client_; -// client_ = nullptr; -// } -// Aws::ShutdownAPI(options_); -// return Status::OK(); -//} -// -//Status -//S3ClientWrapper::CreateBucket(std::string& bucket_name) { -// Aws::S3::Model::CreateBucketRequest request; -// request.SetBucket(bucket_name); -// -// auto outcome = client_->CreateBucket(request); -// -// if (outcome.IsSuccess()) -// { -// return Status::OK(); -// } -// else -// { -// std::cout << "CreateBucket error: " -// << outcome.GetError().GetExceptionName() << std::endl -// << outcome.GetError().GetMessage() << std::endl; -// switch(outcome.GetError().GetErrorType()) { -// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: -// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: -// return Status::AlreadyExist(outcome.GetError().GetMessage()); -// default: -// return Status::Error(outcome.GetError().GetMessage()); -// } -// } -//} -// -//Status -//S3ClientWrapper::DeleteBucket(std::string& bucket_name) { -// Aws::S3::Model::DeleteBucketRequest bucket_request; -// bucket_request.SetBucket(bucket_name); -// -// auto outcome = client_->DeleteBucket(bucket_request); -// -// if (outcome.IsSuccess()) -// { -// return Status::OK(); -// } -// else -// { -// std::cout << "DeleteBucket error: " -// << outcome.GetError().GetExceptionName() << " - " -// << outcome.GetError().GetMessage() << std::endl; -// return Status::Error(outcome.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { -// -// PutObjectRequest putObjectRequest; -// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); -// -// auto input_data = Aws::MakeShared("PutObjectInputStream", -// pathkey.c_str(), -// std::ios_base::in | std::ios_base::binary); -// putObjectRequest.SetBody(input_data); -// auto put_object_result = client_->PutObject(putObjectRequest); -// if (put_object_result.IsSuccess()) { -// return Status::OK(); -// } else { -// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " -// << put_object_result.GetError().GetMessage() << std::endl; -// return Status::Error(put_object_result.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { -// GetObjectRequest object_request; -// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); -// auto get_object_outcome = client_->GetObject(object_request); -// if (get_object_outcome.IsSuccess()) { -// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); -// local_file << get_object_outcome.GetResult().GetBody().rdbuf(); -// return Status::OK(); -// } else { -// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " -// << get_object_outcome.GetError().GetMessage() << std::endl; -// return Status::Error(get_object_outcome.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { -// Aws::S3::Model::DeleteObjectRequest object_request; -// object_request.WithBucket(bucket_name).WithKey(object_key); -// -// auto delete_object_outcome = client_->DeleteObject(object_request); -// -// if (delete_object_outcome.IsSuccess()) { -// return Status::OK(); -// } else { -// std::cout << "DeleteObject error: " << -// delete_object_outcome.GetError().GetExceptionName() << " " << -// delete_object_outcome.GetError().GetMessage() << std::endl; -// -// return Status::Error(delete_object_outcome.GetError().GetMessage()); -// } -//} -// -//} -//} -//} -//} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h deleted file mode 100644 index 95a94065..00000000 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -//#pragma once -// -//#include "storage/IStorage.h" -// -// -//#include -//#include -//#include -// -// -//using namespace Aws::S3; -//using namespace Aws::S3::Model; -// -//namespace zilliz { -//namespace milvus { -//namespace engine { -//namespace storage { -// -//class S3ClientWrapper : public IStorage { -// public: -// -// S3ClientWrapper() = default; -// ~S3ClientWrapper() = default; -// -// Status Create(const std::string &ip_address, -// const std::string &port, -// const std::string &access_key, -// const std::string &secret_key) override; -// Status Close() override; -// -// Status CreateBucket(std::string& bucket_name) override; -// Status DeleteBucket(std::string& bucket_name) override; -// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; -// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; -// Status DeleteFile(std::string &bucket_name, std::string &object_key) override; -// -// private: -// S3Client *client_ = nullptr; -// Aws::SDKOptions options_; -//}; -// -//} -//} -//} -//} \ No newline at end of file diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 3b02016c..ec270c06 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -1,24 +1,16 @@ -ARROW_VERSION=zilliz BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 -FAISS_VERSION=branch-0.1.0 -MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 -JSONCONS_VERSION=0.126.0 -LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 MYSQLPP_VERSION=3.2.4 -OPENBLAS_VERSION=v0.3.6 PROMETHEUS_VERSION=v0.7.0 -ROCKSDB_VERSION=v6.0.2 SNAPPY_VERSION=1.1.7 SQLITE_VERSION=3280000 SQLITE_ORM_VERSION=master YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 -AWS_VERSION=1.7.125 LIBUNWIND_VERSION=1.3.1 GPERFTOOLS_VERSION=2.7 GRPC_VERSION=master diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 03729071..40db37a0 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -17,20 +17,70 @@ # under the License. #------------------------------------------------------------------------------- -link_directories( - "${CMAKE_BINARY_DIR}/lib" -) +include_directories(${MILVUS_SOURCE_DIR}) +include_directories(${MILVUS_ENGINE_SRC}) + +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) +set(db_scheduler_files + ${scheduler_files} + ${scheduler_context_files} + ${scheduler_task_files} + ) + +set(grpc_service_files + grpc/gen-milvus/milvus.grpc.pb.cc + grpc/gen-milvus/milvus.pb.cc + grpc/gen-status/status.grpc.pb.cc + grpc/gen-status/status.pb.cc + ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_root_files) +set(scheduler_files + ${scheduler_action_files} + ${scheduler_event_files} + ${scheduler_job_files} + ${scheduler_resource_files} + ${scheduler_task_files} + ${scheduler_root_files} + ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -set(unittest_srcs +set(unittest_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) -set(require_files +set(helper_files ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ${MILVUS_ENGINE_SRC}/utils/Status.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc ) set(unittest_libs @@ -49,15 +99,16 @@ set(unittest_libs z ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so cudart + cublas ) foreach(dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) endforeach() -add_subdirectory(server) add_subdirectory(db) add_subdirectory(knowhere) add_subdirectory(metrics) add_subdirectory(scheduler) +add_subdirectory(server) #add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 29440dcf..3ab33d3c 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -17,61 +17,27 @@ # under the License. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) - -aux_source_directory(./ test_srcs) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) -set(db_scheduler_srcs +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) + +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + +set(db_test_files + ${cache_files} + ${config_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_scheduler_files} + ${metrics_files} ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) - -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") - -include_directories(/usr/include/mysql) - -set(db_test_src - ${config_files} - ${cache_srcs} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${util_files} - ${require_files} - ${test_srcs} + ${knowhere_files} + ${helper_files} + ${test_files} ) -cuda_add_executable(db_test ${db_test_src}) +cuda_add_executable(db_test ${db_test_files}) set(db_libs sqlite @@ -81,13 +47,7 @@ set(db_libs mysqlpp ) -set(knowhere_libs - knowhere - cudart - cublas - ) - -target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) +target_link_libraries(db_test ${db_libs} knowhere ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 27e2612a..5606b382 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -20,15 +20,15 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -set(knowhere_src +set(knowhere_files ${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) -set(helper +set(helper_files utils.cpp ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ) set(knowhere_libs knowhere @@ -36,7 +36,7 @@ set(knowhere_libs cublas ) -add_executable(knowhere_test knowhere_test.cpp ${knowhere_src} ${helper}) +add_executable(knowhere_test knowhere_test.cpp ${knowhere_files} ${helper_files}) target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs}) install(TARGETS knowhere_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index dddba290..9df30c43 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,86 +17,35 @@ # under the License. #------------------------------------------------------------------------------- -include_directories(../../src) - - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src) -aux_source_directory(./ test_srcs) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) include_directories(/usr/include) -include_directories(../../third_party/build/include) -link_directories(../../third_party/build/lib) -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") -include_directories(../../src/metrics) +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -include_directories(/usr/include/mysql) - -set(count_test_src +set(metrics_test_files + ${cache_files} ${config_files} - ${cache_srcs} ${db_main_files} ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${metrics_src} - ${test_srcs} - ${util_files} + ${db_scheduler_files} + ${metrics_files} + ${scheduler_files} + ${knowhere_files} + ${helper_files} + ${test_files} ) -add_executable(metrics_test ${count_test_src} ${require_files} ) - -set(knowhere_libs - knowhere - ) +add_executable(metrics_test ${metrics_test_files}) target_link_libraries(metrics_test - ${knowhere_libs} - cudart - cublas + knowhere sqlite boost_system_static boost_filesystem_static lz4 - metrics - gtest - pthread - z mysqlpp ${unittest_libs} ) diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 7359420e..9386596d 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -17,62 +17,27 @@ # under the License. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) -aux_source_directory(./ test_srcs) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files}) - -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") - -include_directories(/usr/include/mysql) - -set(scheduler_test_src - ${unittest_srcs} - ${test_srcs} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") +set(scheduler_test_files + ${cache_files} ${config_files} - ${cache_srcs} ${db_main_files} ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${knowhere_src} - ${util_files} - ${require_files} + ${db_scheduler_files} + ${scheduler_files} + ${knowhere_files} + ${helper_files} + ${unittest_files} + ${test_files} ) -cuda_add_executable(scheduler_test ${scheduler_test_src}) +cuda_add_executable(scheduler_test ${scheduler_test_files}) set(scheduler_libs sqlite @@ -82,14 +47,7 @@ set(scheduler_libs mysqlpp ) -set(knowhere_libs - knowhere - cudart - cublas - ) - - -target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) +target_link_libraries(scheduler_test ${scheduler_libs} knowhere ${unittest_libs}) install(TARGETS scheduler_test DESTINATION unittest) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 8e068599..395f2e40 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -17,75 +17,19 @@ # under the License. #------------------------------------------------------------------------------- -include_directories(${MILVUS_ENGINE_SRC}/) -include_directories(/usr/include) - -include_directories(/usr/include/mysql) - -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) - -aux_source_directory(${MILVUS_ENGINE_SRC}/server server_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_src) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) - -aux_source_directory(./ test_srcs) - -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) +include_directories(/usr/include) +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -set(db_src - ${config_files} - ${cache_srcs} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${util_files} - ${require_files} - ${test_srcs} - ) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) -set(utils_srcs +set(helper_files ${helper_files} ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp - ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp - ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp - ) + ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp) set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc @@ -94,44 +38,49 @@ set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) - -cuda_add_executable(server_test - ${db_src} - ${unittest_srcs} - ${grpc_server_src} - ${server_src} - ${utils_srcs} +set(server_test_files + ${cache_files} + ${config_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_scheduler_files} + ${scheduler_files} + ${server_files} + ${grpc_server_files} ${grpc_service_files} - ${require_files} + ${knowhere_files} + ${helper_files} + ${unittest_files} + ${test_files} ) +cuda_add_executable(server_test ${server_test_files}) + +set(client_grpc_lib + grpcpp_channelz + grpc++ + grpc + grpc_protobuf + grpc_protoc) + set(require_libs knowhere stdc++ - cudart - cublas sqlite mysqlpp boost_system_static boost_filesystem_static snappy - z bz2 zstd lz4 - pthread - grpcpp_channelz - grpc++ - grpc - grpc_protobuf - grpc_protoc + ${client_grpc_lib} ) target_link_libraries(server_test ${require_libs} - ${cuda_library} ${unittest_libs} ) diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt deleted file mode 100644 index 79819b2d..00000000 --- a/cpp/unittest/storage/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -#------------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -#------------------------------------------------------------------------------- - -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) - -# Make sure that your call to link_directories takes place before your call to the relevant add_executable. -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(s3_client_test_src - ${unittest_srcs} - ${s3_client_src} - ${require_files} - S3ClientTest.cpp - ${MILVUS_ENGINE_SRC}/db/Status.cpp - ) - -add_executable(s3_test - ${s3_client_test_src} - ${config_files} - ${util_files} - ) - -set(s3_client_libs - stdc++ - aws-cpp-sdk-s3 - aws-cpp-sdk-core - boost_filesystem_static - ) -target_link_libraries(s3_test - ${s3_client_libs} - ${unittest_libs} - curl - crypto) - -install(TARGETS s3_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp deleted file mode 100644 index 540d9ab7..00000000 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// -//#include "storage/IStorage.h" -//#include "storage/s3/S3ClientWrapper.h" -//#include -//#include -//#include -// -// -//using namespace zilliz::milvus::engine; -// -//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { -// -// std::shared_ptr storage_ptr = std::make_shared(); -// -// std::string ip_address = "127.0.0.1"; -// std::string port = "9000"; -// std::string access_key = "AKIAIOSFODNN7EXAMPLE"; -// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; -// -// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); -// ASSERT_TRUE(status.ok()); -// -// std::string filename = "/tmp/s3_test_file"; -// std::string bucket_name = "bucktname"; -// std::string object_name = "test_file"; -// -// status = storage_ptr->CreateBucket(bucket_name); -// std::cout << status.IsAlreadyExist() << std::endl; -// if (status.IsAlreadyExist()) { -// status = storage_ptr->DeleteBucket(bucket_name); -// status = storage_ptr->CreateBucket(bucket_name); -// } -// -// ASSERT_TRUE(status.ok()); -// -// std::ofstream ofile(filename); -// std::stringstream ss; -// for (int i = 0; i < 1024; ++i) { -// ss << i; -// } -// ofile << ss.str() << std::endl; -// ofile.close(); -// status = storage_ptr->UploadFile(bucket_name, object_name, filename); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->DownloadFile(bucket_name, object_name, filename); -// std::ifstream infile(filename); -// std::string in_buffer; -// infile >> in_buffer; -// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); -// -// status = storage_ptr->DeleteFile(bucket_name, object_name); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->DeleteBucket(bucket_name); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->Close(); -// ASSERT_TRUE(status.ok()); -//} -// -- GitLab