Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 4193b2e3 编写于 作者: Y yu yunfeng
浏览文件
操作

Merge remote-tracking branch 'main/branch-1.2' into branch-1.2 


Former-commit-id: 3ec3ca22cca08e517779bc268c535d5edd8768fe
上级 439a5d10 ee8d8100
隐藏空白更改
内联 并排
Showing with 2492 addition and 338 deletion
cpp/CHANGELOG.md
浏览文件 @ 4193b2e3
......@@ -12,9 +12,14 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-5 - Implement Auto Archive Feature
- MS-16 - Implement metrics without prometheus
- MS-6 - Implement SDK interface part 1
- MS-21 - Implement SDK interface part 2
- MS-26 - cmake. Add thirdparty packages
- MS-31 - cmake: add prometheus
### Task
- MS-1 - Add CHANGELOG.md
- MS-4 - Refactor the vecwise_engine code structure
- MS-6 - Implement SDK interface part 1
- MS-20 - Clean Code Part 1
- MS-30 - Use faiss v1.5.2
cpp/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -4,10 +4,33 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.12)
cmake_minimum_required(VERSION 3.14)
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(MEGASEARCH_VERSION "0.1.0")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}")
project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}")
project(vecwise_engine LANGUAGES CUDA CXX)
set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}")
set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}")
set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}")
if(MEGASEARCH_VERSION_MAJOR STREQUAL ""
OR MEGASEARCH_VERSION_MINOR STREQUAL ""
OR MEGASEARCH_VERSION_PATCH STREQUAL "")
message(FATAL_ERROR "Failed to determine MegaSearch version from '${MEGASEARCH_VERSION}'")
endif()
message(STATUS "MegaSearch version: "
"${MEGASEARCH_VERSION_MAJOR}.${MEGASEARCH_VERSION_MINOR}.${MEGASEARCH_VERSION_PATCH} "
"(full: '${MEGASEARCH_VERSION}')")
set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
......@@ -19,6 +42,7 @@ if (GPU_VERSION STREQUAL "ON")
endif ()
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building vecwise_engine on x86 architecture")
......@@ -32,49 +56,52 @@ else()
set(VECWISE_BUILD_ARCH unknown)
endif()
if(DEFINED UNIX)
message("building vecwise on Unix")
set(VECWISE_BUILD_SYSTEM macos)
elseif(DEFINED APPLE)
message("building vecwise on MacOS")
set(VECWISE_BUILD_SYSTEM unix)
else()
message("unknown OS")
set(VECWISE_BUILD_SYSTEM unknown)
endif ()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
if (GPU_VERSION STREQUAL "ON")
set(ENABLE_LICENSE "ON")
add_definitions("-DENABLE_LICENSE")
endif ()
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif()
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
#if (BUILD_UNIT_TEST)
# option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON)
#endif(BUILD_UNIT_TEST)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
include_directories(${MEGASEARCH_SOURCE_DIR})
link_directories(${MEGASEARCH_BINARY_DIR})
## Following should be check
set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
#set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
#set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
add_compile_definitions(PROFILER=${PROFILER})
include_directories(${VECWISE_ENGINE_INCLUDE})
include_directories(${VECWISE_ENGINE_SRC})
include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
#include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
link_directories(${CMAKE_CURRRENT_BINARY_DIR})
link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
#execute_process(COMMAND bash build.sh
# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
add_subdirectory(src)
add_subdirectory(unittest)
if (BUILD_UNIT_TEST)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
......@@ -93,3 +120,5 @@ install(FILES
conf/vecwise_engine_log.conf
DESTINATION
conf)
config_summary()
cpp/build.sh
浏览文件 @ 4193b2e3
......@@ -72,7 +72,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then
make clean
fi
make -j || exit 1
make -j 1 || exit 1
if [[ ${BUILD_TYPE} != "Debug" ]]; then
strip src/vecwise_server
......
cpp/cmake/BuildUtils.cmake 0 → 100644
浏览文件 @ 4193b2e3
function(ADD_THIRDPARTY_LIB LIB_NAME)
set(options)
set(one_value_args SHARED_LIB STATIC_LIB)
set(multi_value_args DEPS INCLUDE_DIRECTORIES)
cmake_parse_arguments(ARG
"${options}"
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
if(NOT ARG_STATIC_LIB)
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_STATIC_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_SHARED_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
else()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endfunction()
\ No newline at end of file
cpp/cmake/DefineOptions.cmake 0 → 100644
浏览文件 @ 4193b2e3
macro(set_option_category name)
set(MEGASEARCH_OPTION_CATEGORY ${name})
list(APPEND "MEGASEARCH_OPTION_CATEGORIES" ${name})
endmacro()
macro(define_option name description default)
option(${name} ${description} ${default})
list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" ${default})
set("${name}_OPTION_TYPE" "bool")
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
macro(define_option_string name description default)
set(${name} ${default} CACHE STRING ${description})
list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" "\"${default}\"")
set("${name}_OPTION_TYPE" "string")
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endmacro()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT "AUTO")
define_option_string(MEGASEARCH_DEPENDENCY_SOURCE
"Method to use for acquiring MEGASEARCH's build dependencies"
"${MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF)
define_option(MEGASEARCH_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
Note that this requires linking Boost statically" ON)
define_option(MEGASEARCH_BOOST_HEADER_ONLY "Use only BOOST headers" OFF)
define_option(MEGASEARCH_WITH_BZ2 "Build with BZ2 compression" ON)
define_option(MEGASEARCH_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON)
define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against"
"-gencode=arch=compute_61,code=sm_61")
define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON)
define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON)
define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON)
define_option(MEGASEARCH_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON)
define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON)
define_option(MEGASEARCH_WITH_SQLITE "Build with SQLite library" ON)
define_option(MEGASEARCH_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
define_option(MEGASEARCH_WITH_THRIFT "Build with Apache Thrift library" ON)
define_option(MEGASEARCH_WITH_YAMLCPP "Build with yaml-cpp library" ON)
define_option(MEGASEARCH_WITH_ZLIB "Build with zlib compression" ON)
if(CMAKE_VERSION VERSION_LESS 3.7)
set(MEGASEARCH_WITH_ZSTD_DEFAULT OFF)
else()
# ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7.
set(MEGASEARCH_WITH_ZSTD_DEFAULT ON)
endif()
define_option(MEGASEARCH_WITH_ZSTD "Build with zstd compression" ${MEGASEARCH_WITH_ZSTD_DEFAULT})
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(MEGASEARCH_USE_STATIC_CRT "Build MEGASEARCH with statically linked CRT" OFF)
endif()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
if (BUILD_UNIT_TEST)
define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON)
else()
define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
message(STATUS "---------------------------------------------------------------------")
message(STATUS "MEGASEARCH version: ${MEGASEARCH_VERSION}")
message(STATUS)
message(STATUS "Build configuration summary:")
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
endif()
foreach(category ${MEGASEARCH_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
set(option_names ${MEGASEARCH_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
set(value "${${name}}")
endif()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
set(description_padding "
")
endif()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach()
endmacro()
cpp/cmake/GoogleTest.cmake 已删除 100644 → 0
浏览文件 @ 439a5d10
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tries to find GTest headers and libraries.
#
# Usage of this module as follows:
#
# find_package(GTest)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# GTest_HOME - When set, this path is inspected instead of standard library
# locations as the root of the GTest installation.
# The environment variable GTEST_HOME overrides this veriable.
#
# This module defines
# GTEST_INCLUDE_DIR, directory containing headers
# GTEST_LIBS, directory containing gtest libraries
# GTEST_STATIC_LIB, path to libgtest.a
# GTEST_SHARED_LIB, path to libgtest's shared library
# GTEST_FOUND, whether gtest has been found
if( NOT "${GTEST_HOME}" STREQUAL "")
file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path )
list( APPEND _gtest_roots ${_native_path} )
elseif ( GTest_HOME )
list( APPEND _gtest_roots ${GTest_HOME} )
endif()
# Try the parameterized roots, if they exist
if ( _gtest_roots )
find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
PATHS ${_gtest_roots} NO_DEFAULT_PATH
PATH_SUFFIXES "include" )
find_library( GTEST_LIBRARIES NAMES gtest gtest_main
PATHS ${_gtest_roots} NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
else ()
find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h )
find_library( GTEST_LIBRARIES NAMES gtest )
endif ()
if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
set(GTEST_FOUND TRUE)
get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
set(GTEST_LIB_NAME gtest)
set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX})
set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
else ()
set(GTEST_FOUND FALSE)
endif ()
if (GTEST_FOUND)
if (NOT GTest_FIND_QUIETLY)
message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}")
endif ()
else ()
if (NOT GTest_FIND_QUIETLY)
set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ")
if ( _gtest_roots )
set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.")
else ()
set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.")
endif ()
if (GTest_FIND_REQUIRED)
message(FATAL_ERROR "${GTEST_ERR_MSG}")
else (GTest_FIND_REQUIRED)
message(STATUS "${GTEST_ERR_MSG}")
endif (GTest_FIND_REQUIRED)
endif ()
endif ()
mark_as_advanced(
GTEST_INCLUDE_DIR
GTEST_LIBS
GTEST_LIBRARIES
GTEST_STATIC_LIB
GTEST_SHARED_LIB
)
cpp/cmake/ThirdPartyPackages.cmake
浏览文件 @ 4193b2e3
......@@ -15,52 +15,1526 @@
# specific language governing permissions and limitations
# under the License.
set(GTEST_VERSION "1.8.0")
set(MEGASEARCH_THIRDPARTY_DEPENDENCIES
message(MEGASEARCH_BUILD_TESTS ${MEGASEARCH_BUILD_TESTS})
BOOST
BZip2
Easylogging++
FAISS
GTest
LAPACK
Lz4
OpenBLAS
Prometheus
RocksDB
Snappy
SQLite
SQLite_ORM
Thrift
yaml-cpp
ZLIB
ZSTD)
if(MEGASEARCH_BUILD_TESTS)
add_custom_target(unittest ctest -L unittest)
message(STATUS "Using ${MEGASEARCH_DEPENDENCY_SOURCE} approach to find dependencies")
if("$ENV{GTEST_HOME}" STREQUAL "")
message("Yes")
set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
# For each dependency, set dependency source to global default, if unset
foreach(DEPENDENCY ${MEGASEARCH_THIRDPARTY_DEPENDENCIES})
if("${${DEPENDENCY}_SOURCE}" STREQUAL "")
set(${DEPENDENCY}_SOURCE ${MEGASEARCH_DEPENDENCY_SOURCE})
endif()
endforeach()
macro(build_dependency DEPENDENCY_NAME)
if("${DEPENDENCY_NAME}" STREQUAL "BZip2")
build_bzip2()
elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++")
build_easyloggingpp()
elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK")
build_lapack()
elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4")
build_lz4()
elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
build_gtest()
elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus")
build_prometheus()
elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB")
build_rocksdb()
elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy")
build_snappy()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
build_sqlite()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM")
build_sqlite_orm()
elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift")
build_thrift()
elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
build_yamlcpp()
elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB")
build_zlib()
elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD")
build_zstd()
else()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
endmacro()
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GTEST_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GTEST_MAIN_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#message(STATUS "Finding ${DEPENDENCY_NAME} package")
# find_package(${DEPENDENCY_NAME} QUIET)
# if (NOT ${DEPENDENCY_NAME}_FOUND)
#message(STATUS "${DEPENDENCY_NAME} package not found")
build_dependency(${DEPENDENCY_NAME})
# endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
find_package(${DEPENDENCY_NAME} REQUIRED)
endif ()
endmacro()
set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}
-DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS})
# ----------------------------------------------------------------------
# ExternalProject options
ExternalProject_Add(googletest
URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" "${GTEST_MAIN_STATIC_LIB}"
CMAKE_ARGS ${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
set(GTEST_VENDORED 1)
string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
if(NOT MSVC)
# Set -fPIC on all external projects
set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
endif()
# CC/CXX environment variables are captured on the first invocation of the
# builder (e.g make or ninja) instead of when CMake is invoked into to build
# directory. This leads to issues if the variables are exported in a subshell
# and the invocation of make/ninja is in distinct subshell without the same
# environment (CC/CXX).
set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
if(CMAKE_AR)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR})
endif()
if(CMAKE_RANLIB)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB})
endif()
# External projects are still able to override the following declarations.
# cmake command line will favor the last defined variable when a duplicate is
# encountered. This requires that `EP_COMMON_CMAKE_ARGS` is always the first
# argument.
set(EP_COMMON_CMAKE_ARGS
${EP_COMMON_TOOLCHAIN}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS})
if(NOT MEGASEARCH_VERBOSE_THIRDPARTY_BUILD)
set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
else()
set(EP_LOG_OPTIONS)
endif()
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
# Using make -j in sub-make is fragile
# see discussion https://github.com/apache/MEGASEARCH/pull/2779
if(${CMAKE_GENERATOR} MATCHES "Makefiles")
set(MAKE_BUILD_ARGS "")
else()
# limit the maximum number of jobs for ninja
set(MAKE_BUILD_ARGS "-j4")
endif()
# ----------------------------------------------------------------------
# Find pthreads
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
# ----------------------------------------------------------------------
# Versions and URLs for toolchain builds, which also can be used to configure
# offline builds
# Read toolchain versions from cpp/thirdparty/versions.txt
set(THIRDPARTY_DIR "${MEGASEARCH_SOURCE_DIR}/thirdparty")
file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT)
foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
# Exclude comments
if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
continue()
endif()
string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY})
string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY})
# Skip blank or malformed lines
if(${_LIB_VERSION} STREQUAL "")
continue()
endif()
# For debugging
#message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}")
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach()
if(DEFINED ENV{MEGASEARCH_BOOST_URL})
set(BOOST_SOURCE_URL "$ENV{MEGASEARCH_BOOST_URL}")
else()
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
set(BOOST_SOURCE_URL
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
)
endif()
if(DEFINED ENV{MEGASEARCH_BZIP2_URL})
set(BZIP2_SOURCE_URL "$ENV{MEGASEARCH_BZIP2_URL}")
else()
set(BZIP2_SOURCE_URL "https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_EASYLOGGINGPP_URL})
set(EASYLOGGINGPP_SOURCE_URL "$ENV{MEGASEARCH_EASYLOGGINGPP_URL}")
else()
set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{MEGASEARCH_FAISS_URL}")
else()
set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz")
endif()
if (DEFINED ENV{MEGASEARCH_GTEST_URL})
set(GTEST_SOURCE_URL "$ENV{MEGASEARCH_GTEST_URL}")
else ()
set(GTEST_SOURCE_URL
"https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_LAPACK_URL})
set(LAPACK_SOURCE_URL "$ENV{MEGASEARCH_LAPACK_URL}")
else()
set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_LZ4_URL})
set(LZ4_SOURCE_URL "$ENV{MEGASEARCH_LZ4_URL}")
else()
set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz")
endif()
if (DEFINED ENV{MEGASEARCH_OPENBLAS_URL})
set(OPENBLAS_SOURCE_URL "$ENV{MEGASEARCH_OPENBLAS_URL}")
else ()
set(OPENBLAS_SOURCE_URL
"https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz")
endif()
if (DEFINED ENV{MEGASEARCH_PROMETHEUS_URL})
set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}")
else ()
set(PROMETHEUS_SOURCE_URL
"https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz")
endif()
if (DEFINED ENV{MEGASEARCH_ROCKSDB_URL})
set(ROCKSDB_SOURCE_URL "$ENV{MEGASEARCH_ROCKSDB_URL}")
else ()
set(ROCKSDB_SOURCE_URL
"https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_SNAPPY_URL})
set(SNAPPY_SOURCE_URL "$ENV{MEGASEARCH_SNAPPY_URL}")
else()
set(SNAPPY_SOURCE_URL
"https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_SQLITE_URL})
set(SQLITE_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_URL}")
else()
set(SQLITE_SOURCE_URL
"https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_SQLITE_ORM_URL})
set(SQLITE_ORM_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_ORM_URL}")
else()
set(SQLITE_ORM_SOURCE_URL
"https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip")
endif()
if(DEFINED ENV{MEGASEARCH_THRIFT_URL})
set(THRIFT_SOURCE_URL "$ENV{MEGASEARCH_THRIFT_URL}")
else()
set(THRIFT_SOURCE_URL
"http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_YAMLCPP_URL})
set(YAMLCPP_SOURCE_URL "$ENV{MEGASEARCH_YAMLCPP_URL}")
else()
set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_ZLIB_URL})
set(ZLIB_SOURCE_URL "$ENV{MEGASEARCH_ZLIB_URL}")
else()
set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MEGASEARCH_ZSTD_URL})
set(ZSTD_SOURCE_URL "$ENV{MEGASEARCH_ZSTD_URL}")
else()
set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz")
endif()
# ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
set(Boost_USE_MULTITHREADED ON)
if(MSVC AND MEGASEARCH_USE_STATIC_CRT)
set(Boost_USE_STATIC_RUNTIME ON)
endif()
set(Boost_ADDITIONAL_VERSIONS
"1.70.0"
"1.70"
"1.69.0"
"1.69"
"1.68.0"
"1.68"
"1.67.0"
"1.67"
"1.66.0"
"1.66"
"1.65.0"
"1.65"
"1.64.0"
"1.64"
"1.63.0"
"1.63"
"1.62.0"
"1.61"
"1.61.0"
"1.62"
"1.60.0"
"1.60")
if(MEGASEARCH_BOOST_VENDORED)
set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep")
set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
set(BOOST_BUILD_LINK "static")
set(BOOST_STATIC_SYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_FILESYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_SERIALIZATION_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_SYSTEM_LIBRARY boost_system_static)
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static)
if(MEGASEARCH_BOOST_HEADER_ONLY)
set(BOOST_BUILD_PRODUCTS)
set(BOOST_CONFIGURE_COMMAND "")
set(BOOST_BUILD_COMMAND "")
else()
find_package(GTest REQUIRED)
set(GTEST_VENDORED 0)
set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY}
${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY})
set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}"
"--with-libraries=filesystem,serialization,system")
if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_BUILD_VARIANT "debug")
else()
set(BOOST_BUILD_VARIANT "release")
endif()
set(BOOST_BUILD_COMMAND
"./b2"
"link=${BOOST_BUILD_LINK}"
"variant=${BOOST_BUILD_VARIANT}"
"cxxflags=-fPIC")
add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}")
add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}")
set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY})
endif()
externalproject_add(boost_ep
URL
${BOOST_SOURCE_URL}
BUILD_BYPRODUCTS
${BOOST_BUILD_PRODUCTS}
BUILD_IN_SOURCE
1
CONFIGURE_COMMAND
${BOOST_CONFIGURE_COMMAND}
BUILD_COMMAND
${BOOST_BUILD_COMMAND}
INSTALL_COMMAND
""
${EP_LOG_OPTIONS})
set(Boost_INCLUDE_DIR "${BOOST_PREFIX}")
set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
add_dependencies(boost_system_static boost_ep)
add_dependencies(boost_filesystem_static boost_ep)
add_dependencies(boost_serialization_static boost_ep)
message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}")
message(STATUS "GTest static library: ${GTEST_STATIC_LIB}")
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
else()
if(MSVC)
# disable autolinking in boost
add_definitions(-DBOOST_ALL_NO_LIB)
endif()
# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
# # In older versions of CMake (such as 3.2), the system paths for Boost will
# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT
# set(Boost_NO_SYSTEM_PATHS ON)
# endif()
if(MEGASEARCH_BOOST_USE_SHARED)
# Find shared Boost libraries.
set(Boost_USE_STATIC_LIBS OFF)
set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
set(BUILD_SHARED_LIBS ON)
if(MSVC)
# force all boost libraries to dynamic link
add_definitions(-DBOOST_ALL_DYN_LINK)
endif()
if(MEGASEARCH_BOOST_HEADER_ONLY)
find_package(Boost REQUIRED)
else()
find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
set(BOOST_SYSTEM_LIBRARY Boost::system)
set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
endif()
set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
unset(BUILD_SHARED_LIBS_KEEP)
else()
# Find static boost headers and libs
# TODO Differentiate here between release and debug builds
set(Boost_USE_STATIC_LIBS ON)
if(MEGASEARCH_BOOST_HEADER_ONLY)
find_package(Boost REQUIRED)
else()
find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
set(BOOST_SYSTEM_LIBRARY Boost::system)
set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
endif()
endif()
endif()
#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR})
#message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
link_directories(SYSTEM ${BOOST_LIB_DIR})
# ----------------------------------------------------------------------
# bzip2
macro(build_bzip2)
message(STATUS "Building BZip2-${BZIP2_VERSION} from source")
set(BZIP2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix/src/bzip2_ep")
set(BZIP2_INCLUDE_DIR "${BZIP2_PREFIX}/include")
set(BZIP2_STATIC_LIB
"${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}")
externalproject_add(bzip2_ep
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
CFLAGS=${EP_C_FLAGS}
INSTALL_COMMAND
${MAKE}
install
PREFIX=${BZIP2_PREFIX}
CFLAGS=${EP_C_FLAGS}
INSTALL_DIR
${BZIP2_PREFIX}
URL
${BZIP2_SOURCE_URL}
BUILD_BYPRODUCTS
"${BZIP2_STATIC_LIB}")
file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}")
add_library(bzip2 STATIC IMPORTED)
set_target_properties(
bzip2
PROPERTIES IMPORTED_LOCATION "${BZIP2_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}")
add_dependencies(bzip2 bzip2_ep)
endmacro()
if(MEGASEARCH_WITH_BZ2)
resolve_dependency(BZip2)
if(NOT TARGET bzip2)
add_library(bzip2 UNKNOWN IMPORTED)
set_target_properties(bzip2
PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}"
INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}")
endif()
link_directories(SYSTEM ${BZIP2_PREFIX}/lib/)
include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}")
endif()
# ----------------------------------------------------------------------
# Easylogging++
macro(build_easyloggingpp)
message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source")
set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep")
set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include")
set(EASYLOGGINGPP_STATIC_LIB
"${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(EASYLOGGINGPP_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib
-Dtest=OFF
-Dbuild_static_lib=ON)
externalproject_add(easyloggingpp_ep
URL
${EASYLOGGINGPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${EASYLOGGINGPP_CMAKE_ARGS}
BUILD_BYPRODUCTS
${EASYLOGGINGPP_STATIC_LIB})
file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}")
add_library(easyloggingpp STATIC IMPORTED)
set_target_properties(
easyloggingpp
PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}")
add_dependencies(easyloggingpp easyloggingpp_ep)
endmacro()
if(MEGASEARCH_WITH_EASYLOGGINGPP)
resolve_dependency(Easylogging++)
get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib")
include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}")
endif()
# ----------------------------------------------------------------------
# OpenBLAS
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
INSTALL_COMMAND
${MAKE}
PREFIX=${OPENBLAS_PREFIX}
install
BUILD_BYPRODUCTS
${OPENBLAS_STATIC_LIB})
file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
add_library(openblas STATIC IMPORTED)
set_target_properties(
openblas
PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
add_dependencies(openblas openblas_ep)
endmacro()
#if(MEGASEARCH_WITH_OPENBLAS)
# resolve_dependency(OpenBLAS)
#
# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
#endif()
# ----------------------------------------------------------------------
# LAPACK
macro(build_lapack)
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
set(LAPACK_STATIC_LIB
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(LAPACK_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib)
externalproject_add(lapack_ep
URL
${LAPACK_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${LAPACK_CMAKE_ARGS}
BUILD_BYPRODUCTS
${LAPACK_STATIC_LIB})
file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}")
add_library(lapack STATIC IMPORTED)
set_target_properties(
lapack
PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}")
add_dependencies(lapack lapack_ep)
endmacro()
#if(MEGASEARCH_WITH_LAPACK)
# resolve_dependency(LAPACK)
#
# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
#endif()
# ----------------------------------------------------------------------
# FAISS
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
# add_custom_target(faiss_dependencies)
# add_dependencies(faiss_dependencies openblas_ep)
# add_dependencies(faiss_dependencies openblas)
# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION)
# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY)
set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}"
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS}"
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack"
--without-python)
# if(OPENBLAS_STATIC_LIB)
# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB})
# else()
# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB})
# endif()
# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY})
if(${MEGASEARCH_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=${MEGASEARCH_FAISS_GPU_ARCH}")
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
endif()
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
# BINARY_DIR
# ${FAISS_PREFIX}
# INSTALL_DIR
# ${FAISS_PREFIX}
# BUILD_COMMAND
# ${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
# INSTALL_DIR
# ${FAISS_PREFIX}
# INSTALL_COMMAND
# ""
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
# DEPENDS
# ${faiss_dependencies})
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep)
ExternalProject_Add_StepDependencies(faiss_ep build lapack_ep)
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss STATIC IMPORTED)
set_target_properties(
faiss
PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}")
add_dependencies(faiss faiss_ep)
endmacro()
if(MEGASEARCH_WITH_FAISS)
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
resolve_dependency(LAPACK)
get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
link_directories(SYSTEM "${LAPACK_PREFIX}/lib")
resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
endif()
# ----------------------------------------------------------------------
# Google gtest
macro(build_gtest)
message(STATUS "Building gtest-${GTEST_VERSION} from source")
set(GTEST_VENDORED TRUE)
set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
if(APPLE)
set(GTEST_CMAKE_CXX_FLAGS
${GTEST_CMAKE_CXX_FLAGS}
-DGTEST_USE_OWN_TR1_TUPLE=1
-Wno-unused-value
-Wno-ignored-attributes)
endif()
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GTEST_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GTEST_MAIN_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GTEST_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}"
"-DCMAKE_INSTALL_LIBDIR=lib"
-DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
-DCMAKE_BUILD_TYPE=Release)
set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GMOCK_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
BUILD_BYPRODUCTS
${GTEST_STATIC_LIB}
${GTEST_MAIN_STATIC_LIB}
${GMOCK_STATIC_LIB}
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
add_library(gtest STATIC IMPORTED)
set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB})
set_target_properties(gtest
PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
add_library(gtest_main STATIC IMPORTED)
set_target_properties(gtest_main PROPERTIES IMPORTED_LOCATION
${GTEST_MAIN_STATIC_LIB})
set_target_properties(gtest_main
PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
add_library(gmock STATIC IMPORTED)
set_target_properties(gmock
PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
if(GTEST_VENDORED)
add_dependencies(gtest googletest)
add_dependencies(gtest_main googletest)
add_dependencies(gtest googletest_ep)
add_dependencies(gtest_main googletest_ep)
add_dependencies(gmock googletest_ep)
endmacro()
if (MEGASEARCH_BUILD_TESTS)
#message(STATUS "Resolving gtest dependency")
resolve_dependency(GTest)
if(NOT GTEST_VENDORED)
endif()
# TODO: Don't use global includes but rather target_include_directories
get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM "${GTEST_PREFIX}/lib")
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# lz4
macro(build_lz4)
message(STATUS "Building lz4-${LZ4_VERSION} from source")
set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep")
set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/")
if(MSVC)
if(MEGASEARCH_USE_STATIC_CRT)
if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
else()
set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
endif()
endif()
set(LZ4_STATIC_LIB
"${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib")
set(LZ4_BUILD_COMMAND
BUILD_COMMAND
msbuild.exe
/m
/p:Configuration=${CMAKE_BUILD_TYPE}
/p:Platform=x64
/p:PlatformToolset=v140
${LZ4_RUNTIME_LIBRARY_LINKAGE}
/t:Build
${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln)
else()
set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a")
#set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh
# "AR=${CMAKE_AR}")
set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS})
endif()
# We need to copy the header in lib to directory outside of the build
externalproject_add(lz4_ep
URL
${LZ4_SOURCE_URL}
${EP_LOG_OPTIONS}
UPDATE_COMMAND
${CMAKE_COMMAND}
-E
copy_directory
"${LZ4_BUILD_DIR}/lib"
"${LZ4_PREFIX}/include"
${LZ4_PATCH_COMMAND}
CONFIGURE_COMMAND
""
INSTALL_COMMAND
""
BINARY_DIR
${LZ4_BUILD_DIR}
BUILD_BYPRODUCTS
${LZ4_STATIC_LIB}
${LZ4_BUILD_COMMAND})
file(MAKE_DIRECTORY "${LZ4_PREFIX}/include")
add_library(lz4 STATIC IMPORTED)
set_target_properties(lz4
PROPERTIES IMPORTED_LOCATION "${LZ4_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include")
add_dependencies(lz4 lz4_ep)
endmacro()
if(MEGASEARCH_WITH_LZ4)
resolve_dependency(Lz4)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/)
include_directories(SYSTEM ${LZ4_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# Prometheus
macro(build_prometheus)
message(STATUS "Building Prometheus-${PROMETHEUS_VERSION} from source")
set(PROMETHEUS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix/src/prometheus_ep")
set(PROMETHEUS_STATIC_LIB_NAME prometheus-cpp)
set(PROMETHEUS_CORE_STATIC_LIB
"${PROMETHEUS_PREFIX}/core/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-core${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(PROMETHEUS_PUSH_STATIC_LIB
"${PROMETHEUS_PREFIX}/push/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-push${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(PROMETHEUS_PULL_STATIC_LIB
"${PROMETHEUS_PREFIX}/pull/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-pull${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(PROMETHEUS_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_INSTALL_LIBDIR=lib
-DBUILD_SHARED_LIBS=OFF
"-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}")
externalproject_add(prometheus_ep
URL
${PROMETHEUS_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${PROMETHEUS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
"DESTDIR=${PROMETHEUS_PREFIX}"
install
BUILD_BYPRODUCTS
"${PROMETHEUS_CORE_STATIC_LIB}"
"${PROMETHEUS_PUSH_STATIC_LIB}"
"${PROMETHEUS_PULL_STATIC_LIB}")
#file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/include")
add_library(prometheus-cpp-push STATIC IMPORTED)
set_target_properties(prometheus-cpp-push
PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PUSH_STATIC_LIB}")
# INTERFACE_INCLUDE_DIRECTORIES
# "${PROMETHEUS_PREFIX}/push/include")
add_dependencies(prometheus-cpp-push prometheus_ep)
add_library(prometheus-cpp-pull STATIC IMPORTED)
set_target_properties(prometheus-cpp-pull
PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PULL_STATIC_LIB}")
# INTERFACE_INCLUDE_DIRECTORIES
# "${PROMETHEUS_PREFIX}/pull/include")
add_dependencies(prometheus-cpp-pull prometheus_ep)
add_library(prometheus-cpp-core STATIC IMPORTED)
set_target_properties(prometheus-cpp-core
PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_CORE_STATIC_LIB}")
# INTERFACE_INCLUDE_DIRECTORIES
# "${PROMETHEUS_PREFIX}/core/include")
add_dependencies(prometheus-cpp-core prometheus_ep)
endmacro()
if(MEGASEARCH_WITH_PROMETHEUS)
resolve_dependency(Prometheus)
# TODO: Don't use global includes but rather target_include_directories
#get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES)
#get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/)
include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include)
#get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/)
include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include)
link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/)
include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include)
endif()
# ----------------------------------------------------------------------
# RocksDB
macro(build_rocksdb)
message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source")
set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep")
set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include")
set(ROCKSDB_STATIC_LIB_NAME rocksdb)
set(ROCKSDB_STATIC_LIB
"${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include")
add_library(rocksdb STATIC IMPORTED)
set_target_properties(rocksdb
PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}")
add_dependencies(rocksdb rocksdb_ep)
endmacro()
if(MEGASEARCH_WITH_ROCKSDB)
resolve_dependency(RocksDB)
# TODO: Don't use global includes but rather target_include_directories
# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/)
include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS})
endif()
# ----------------------------------------------------------------------
# Snappy
macro(build_snappy)
message(STATUS "Building snappy-${SNAPPY_VERSION} from source")
set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep-prefix/src/snappy_ep")
set(SNAPPY_STATIC_LIB_NAME snappy)
set(SNAPPY_STATIC_LIB
"${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(SNAPPY_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_INSTALL_LIBDIR=lib
-DSNAPPY_BUILD_TESTS=OFF
"-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}")
externalproject_add(snappy_ep
${EP_LOG_OPTIONS}
BUILD_IN_SOURCE
1
INSTALL_DIR
${SNAPPY_PREFIX}
URL
${SNAPPY_SOURCE_URL}
CMAKE_ARGS
${SNAPPY_CMAKE_ARGS}
BUILD_BYPRODUCTS
"${SNAPPY_STATIC_LIB}")
file(MAKE_DIRECTORY "${SNAPPY_PREFIX}/include")
add_library(snappy STATIC IMPORTED)
set_target_properties(snappy
PROPERTIES IMPORTED_LOCATION "${SNAPPY_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES
"${SNAPPY_PREFIX}/include")
add_dependencies(snappy snappy_ep)
endmacro()
if(MEGASEARCH_WITH_SNAPPY)
# if(Snappy_SOURCE STREQUAL "AUTO")
# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy
# # errornously places them in ${CMAKE_ROOT}/Modules/
# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong
# # location.
# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727
# # https://src.fedoraproject.org/rpms/snappy/pull-request/1
# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/")
# if(NOT Snappy_FOUND)
# find_package(SnappyAlt)
# endif()
# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND)
# build_snappy()
# endif()
# elseif(Snappy_SOURCE STREQUAL "BUNDLED")
# build_snappy()
# elseif(Snappy_SOURCE STREQUAL "SYSTEM")
# # SnappyConfig.cmake is not installed on Ubuntu/Debian
# # TODO: Make a bug report upstream
# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/")
# if(NOT Snappy_FOUND)
# find_package(SnappyAlt REQUIRED)
# endif()
# endif()
resolve_dependency(Snappy)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/)
include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS})
endif()
# ----------------------------------------------------------------------
# SQLite
macro(build_sqlite)
message(STATUS "Building SQLITE-${SQLITE_VERSION} from source")
set(SQLITE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix/src/sqlite_ep")
set(SQLITE_INCLUDE_DIR "${SQLITE_PREFIX}/include")
set(SQLITE_STATIC_LIB
"${SQLITE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite3${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(SQLITE_CONFIGURE_ARGS
"--prefix=${SQLITE_PREFIX}"
"CC=${CMAKE_C_COMPILER}"
"CXX=${CMAKE_CXX_COMPILER}"
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS}")
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${SQLITE_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
"${SQLITE_STATIC_LIB}")
file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}")
add_library(sqlite STATIC IMPORTED)
set_target_properties(
sqlite
PROPERTIES IMPORTED_LOCATION "${SQLITE_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_INCLUDE_DIR}")
add_dependencies(sqlite sqlite_ep)
endmacro()
if(MEGASEARCH_WITH_SQLITE)
resolve_dependency(SQLite)
include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}")
link_directories(SYSTEM ${SQLITE_PREFIX}/lib/)
endif()
# ----------------------------------------------------------------------
# SQLite_ORM
macro(build_sqlite_orm)
message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source")
set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix")
set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz") #sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz
if (NOT EXISTS ${SQLITE_ORM_TAR_NAME})
file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX})
file(DOWNLOAD https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz
${SQLITE_ORM_TAR_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME}
WORKING_DIRECTORY ${SQLITE_ORM_PREFIX})
endif ()
set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm")
#set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep")
#set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm")
# set(SQLITE_ORM_STATIC_LIB
# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}")
#
# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14")
# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14")
#
# set(SQLITE_ORM_CMAKE_ARGS
# ${EP_COMMON_CMAKE_ARGS}
# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}"
# #"LDFLAGS=-L${SQLITE_PREFIX}"
# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include"
# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include"
# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}"
# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}"
# -DSqliteOrm_BuildTests=off
# -DBUILD_TESTING=off)
# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}")
#
# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}")
# externalproject_add(sqlite_orm_ep
# URL
# ${SQLITE_ORM_SOURCE_URL}
# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix
# CONFIGURE_COMMAND
# ""
# BUILD_COMMAND
# ""
# INSTALL_COMMAND
# ""
#${EP_LOG_OPTIONS}
#${EP_LOG_OPTIONS}
# CMAKE_ARGS
# ${SQLITE_ORM_CMAKE_ARGS}
# BUILD_COMMAND
# ${MAKE}
# ${MAKE_BUILD_ARGS}
# #"LDFLAGS=-L${SQLITE_PREFIX}"
# BUILD_IN_SOURCE
# 1
# BUILD_BYPRODUCTS
# "${SQLITE_ORM_STATIC_LIB}"
# )
# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep)
#set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h)
# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}")
# add_library(sqlite_orm STATIC IMPORTED)
## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}")
# set_target_properties(
# sqlite_orm
# PROPERTIES
# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}"
# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}")
# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR})
# target_link_libraries(sqlite_orm INTERFACE sqlite)
#
# add_dependencies(sqlite_orm sqlite_orm_ep)
endmacro()
if(MEGASEARCH_WITH_SQLITE_ORM)
resolve_dependency(SQLite_ORM)
# ExternalProject_Get_Property(sqlite_orm_ep source_dir)
# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep)
include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}")
#message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}")
endif()
# ----------------------------------------------------------------------
# Thrift
macro(build_thrift)
message(STATUS "Building Apache Thrift-${THRIFT_VERSION} from source")
set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep")
set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift")
set(THRIFT_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}"
"-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib"
-DBUILD_SHARED_LIBS=OFF
-DBUILD_TESTING=OFF
-DBUILD_EXAMPLES=OFF
-DBUILD_TUTORIALS=OFF
-DWITH_QT4=OFF
-DWITH_C_GLIB=OFF
-DWITH_JAVA=OFF
-DWITH_PYTHON=OFF
-DWITH_HASKELL=OFF
-DWITH_CPP=ON
-DWITH_STATIC_LIB=ON
-DWITH_LIBEVENT=OFF
-DCMAKE_BUILD_TYPE=Release)
# Thrift also uses boost. Forward important boost settings if there were ones passed.
if(DEFINED BOOST_ROOT)
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}")
endif()
if(DEFINED Boost_NAMESPACE)
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
endif()
set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift")
if(MSVC)
if(MEGASEARCH_USE_STATIC_CRT)
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}")
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON")
else()
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}")
set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF")
endif()
endif()
if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}")
endif()
set(THRIFT_STATIC_LIB
"${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
if(ZLIB_SHARED_LIB)
set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" ${THRIFT_CMAKE_ARGS})
else()
set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" ${THRIFT_CMAKE_ARGS})
endif()
set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY})
if(MSVC)
set(WINFLEXBISON_VERSION 2.4.9)
set(WINFLEXBISON_PREFIX
"${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install")
externalproject_add(
winflexbison_ep
URL
https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
URL_HASH
MD5=a2e979ea9928fbf8567e995e9c0df765
SOURCE_DIR
${WINFLEXBISON_PREFIX}
CONFIGURE_COMMAND
""
BUILD_COMMAND
""
INSTALL_COMMAND
""
${EP_LOG_OPTIONS})
set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep)
set(THRIFT_CMAKE_ARGS
"-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe"
"-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe"
"-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}"
"-DWITH_SHARED_LIB=OFF"
"-DWITH_PLUGIN=OFF"
${THRIFT_CMAKE_ARGS})
elseif(APPLE)
# Some other process always resets BISON_EXECUTABLE to the system default,
# thus we use our own variable here.
if(NOT DEFINED THRIFT_BISON_EXECUTABLE)
find_package(BISON 2.5.1)
# In the case where we cannot find a system-wide installation, look for
# homebrew and ask for its bison installation.
if(NOT BISON_FOUND)
find_program(BREW_BIN brew)
if(BREW_BIN)
execute_process(COMMAND ${BREW_BIN} --prefix bison
OUTPUT_VARIABLE BISON_PREFIX
OUTPUT_STRIP_TRAILING_WHITESPACE)
set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison")
find_package(BISON 2.5.1)
set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}")
endif()
else()
set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}")
endif()
endif()
set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}"
${THRIFT_CMAKE_ARGS})
endif()
externalproject_add(thrift_ep
URL
${THRIFT_SOURCE_URL}
BUILD_BYPRODUCTS
"${THRIFT_STATIC_LIB}"
"${THRIFT_COMPILER}"
CMAKE_ARGS
${THRIFT_CMAKE_ARGS}
DEPENDS
${THRIFT_DEPENDENCIES}
${EP_LOG_OPTIONS})
add_library(thrift STATIC IMPORTED)
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${THRIFT_INCLUDE_DIR}")
set_target_properties(thrift
PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}")
add_dependencies(thrift thrift_ep)
endmacro()
if(MEGASEARCH_WITH_THRIFT)
resolve_dependency(Thrift)
# TODO: Don't use global includes but rather target_include_directories
# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/)
link_directories(SYSTEM ${THRIFT_PREFIX}/lib/)
include_directories(SYSTEM ${THRIFT_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# yaml-cpp
macro(build_yamlcpp)
message(STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source")
set(YAMLCPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix/src/yaml-cpp_ep")
set(YAMLCPP_STATIC_LIB "${YAMLCPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(YAMLCPP_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DCMAKE_INSTALL_PREFIX=${YAMLCPP_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib
-DYAML_CPP_BUILD_TESTS=OFF
-DYAML_CPP_BUILD_TOOLS=OFF)
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_BYPRODUCTS
"${YAMLCPP_STATIC_LIB}"
CMAKE_ARGS
${YAMLCPP_CMAKE_ARGS})
file(MAKE_DIRECTORY "${YAMLCPP_PREFIX}/include")
add_library(yaml-cpp STATIC IMPORTED)
set_target_properties(yaml-cpp
PROPERTIES IMPORTED_LOCATION "${YAMLCPP_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${YAMLCPP_PREFIX}/include")
add_dependencies(yaml-cpp yaml-cpp_ep)
endmacro()
if(MEGASEARCH_WITH_YAMLCPP)
resolve_dependency(yaml-cpp)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/)
include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# zlib
macro(build_zlib)
message(STATUS "Building ZLIB-${ZLIB_VERSION} from source")
set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep")
if(MSVC)
if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
else()
set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
endif()
else()
set(ZLIB_STATIC_LIB_NAME libz.a)
endif()
endif()
\ No newline at end of file
set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}")
set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
-DBUILD_SHARED_LIBS=OFF)
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_BYPRODUCTS
"${ZLIB_STATIC_LIB}"
CMAKE_ARGS
${ZLIB_CMAKE_ARGS})
file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include")
add_library(zlib STATIC IMPORTED)
set_target_properties(zlib
PROPERTIES IMPORTED_LOCATION "${ZLIB_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include")
add_dependencies(zlib zlib_ep)
endmacro()
if(MEGASEARCH_WITH_ZLIB)
resolve_dependency(ZLIB)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# zstd
macro(build_zstd)
message(STATUS "Building zstd-${ZSTD_VERSION} from source")
set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
set(ZSTD_CMAKE_ARGS
${EP_COMMON_TOOLCHAIN}
"-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}"
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR}
-DZSTD_BUILD_PROGRAMS=off
-DZSTD_BUILD_SHARED=off
-DZSTD_BUILD_STATIC=on
-DZSTD_MULTITHREAD_SUPPORT=off)
if(MSVC)
set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib")
if(MEGASEARCH_USE_STATIC_CRT)
set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on")
endif()
else()
set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a")
# Only pass our C flags on Unix as on MSVC it leads to a
# "incompatible command-line options" error
set(ZSTD_CMAKE_ARGS
${ZSTD_CMAKE_ARGS}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS})
endif()
if(CMAKE_VERSION VERSION_LESS 3.7)
message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7")
endif()
externalproject_add(zstd_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ZSTD_CMAKE_ARGS}
SOURCE_SUBDIR
"build/cmake"
INSTALL_DIR
${ZSTD_PREFIX}
URL
${ZSTD_SOURCE_URL}
BUILD_BYPRODUCTS
"${ZSTD_STATIC_LIB}")
file(MAKE_DIRECTORY "${ZSTD_PREFIX}/include")
add_library(zstd STATIC IMPORTED)
set_target_properties(zstd
PROPERTIES IMPORTED_LOCATION "${ZSTD_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_PREFIX}/include")
add_dependencies(zstd zstd_ep)
endmacro()
if(MEGASEARCH_WITH_ZSTD)
resolve_dependency(ZSTD)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${ZSTD_PREFIX}/lib)
include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
endif()
cpp/conf/server_config.yaml
浏览文件 @ 4193b2e3
server_config:
address: 0.0.0.0
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
transfer_protocol: binary #optional: binary, compact, json
server_mode: thread_pool #optional: simple, thread_pool
gpu_index: 0 #which gpu to be used
......
cpp/conf/server_config_template.yaml
浏览文件 @ 4193b2e3
server_config:
address: 0.0.0.0
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
transfer_protocol: binary #optional: binary, compact, json
server_mode: thread_pool #optional: simple, thread_pool
gpu_index: 0 #which gpu to be used
......
cpp/src/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -47,11 +47,52 @@ set(vecwise_engine_files
set(get_sys_info_files
license/GetSysInfo.cpp)
include_directories(/usr/include)
include_directories(/usr/local/cuda/include)
include_directories(thrift/gen-cpp)
#target_link_libraries(megasearch boost_system_static)
#target_link_libraries(megasearch boost_filesystem_static)
#target_link_libraries(megasearch boost_serialization_static)
#target_link_libraries(megasearch bzip2)
#target_link_libraries(megasearch easyloggingpp)
#target_link_libraries(megasearch faiss)
#target_link_libraries(megasearch gtest)
#target_link_libraries(megasearch lapack)
#target_link_libraries(megasearch lz4)
#target_link_libraries(megasearch openblas)
#target_link_libraries(megasearch rocksdb)
#target_link_libraries(megasearch snappy)
#target_link_libraries(megasearch sqlite)
#target_link_libraries(megasearch sqlite_orm)
#target_link_libraries(megasearch thrift)
#target_link_libraries(megasearch yaml-cpp)
#target_link_libraries(megasearch zlib)
#target_link_libraries(megasearch zstd)
set(third_party_libs
easyloggingpp
sqlite
# sqlite_orm
thrift
yaml-cpp
faiss
lapack
openblas
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
rocksdb
boost_system_static
boost_filesystem_static
boost_serialization_static
bzip2
lz4
snappy
zlib
zstd
)
if (GPU_VERSION STREQUAL "ON")
link_directories(/usr/local/cuda/lib64)
set(engine_libs
......@@ -100,13 +141,13 @@ endif ()
cuda_add_library(vecwise_engine STATIC ${vecwise_engine_files})
target_link_libraries(vecwise_engine ${engine_libs})
target_link_libraries(vecwise_engine ${engine_libs} ${third_party_libs})
add_library(metrics STATIC ${metrics_files})
if (ENABLE_LICENSE STREQUAL "ON")
add_library(vecwise_license STATIC ${license_check_files})
target_link_libraries(vecwise_license ${license_libs})
target_link_libraries(vecwise_license ${license_libs} ${third_party_libs})
endif ()
#set(metrics_lib
......@@ -118,17 +159,6 @@ endif ()
#add_library(vecwise_engine STATIC ${metrics_files} )
#target_link_libraries(metrics ${metrics_lib})
add_executable(vecwise_server
${config_files}
${server_files}
${utils_files}
${service_files}
${metrics_files}
${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc
)
set(server_libs
vecwise_engine
librocksdb.a
......@@ -146,16 +176,30 @@ set(server_libs
metrics
)
add_executable(vecwise_server
${config_files}
${server_files}
${utils_files}
${service_files}
${metrics_files}
#${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc
)
if (ENABLE_LICENSE STREQUAL "ON")
target_link_libraries(vecwise_server ${server_libs} vecwise_license)
target_link_libraries(vecwise_server ${server_libs} vecwise_license ${third_party_libs})
else ()
target_link_libraries(vecwise_server ${server_libs})
target_link_libraries(vecwise_server ${server_libs} ${third_party_libs})
endif()
if (ENABLE_LICENSE STREQUAL "ON")
add_executable(get_sys_info ${get_sys_info_files})
add_executable(license_generator ${license_generator_files})
target_link_libraries(get_sys_info ${license_libs} vecwise_license)
target_link_libraries(license_generator ${license_libs})
target_link_libraries(get_sys_info ${license_libs} vecwise_license ${third_party_libs})
target_link_libraries(license_generator ${license_libs} ${third_party_libs})
install(TARGETS get_sys_info DESTINATION bin)
install(TARGETS license_generator DESTINATION bin)
endif ()
install(TARGETS vecwise_server DESTINATION bin)
......
cpp/src/sdk/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -28,8 +28,7 @@ add_library(megasearch_sdk STATIC
link_directories(../../third_party/build/lib)
target_link_libraries(megasearch_sdk
libthrift.a
pthread
${third_party_libs}
)
add_subdirectory(examples)
cpp/src/sdk/examples/simple/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -8,14 +8,12 @@ aux_source_directory(src src_files)
include_directories(src)
include_directories(../../megasearch_sdk/include)
include_directories(/usr/include)
link_directories(${CMAKE_BINARY_DIR}/megasearch_sdk)
add_executable(sdk_simple
./main.cpp
${src_files}
${service_files}
)
target_link_libraries(sdk_simple
......
cpp/src/sdk/examples/simple/main.cpp
浏览文件 @ 4193b2e3
......@@ -19,7 +19,8 @@ main(int argc, char *argv[]) {
printf("Client start...\n");
std::string app_name = basename(argv[0]);
static struct option long_options[] = {{"conf_file", optional_argument, 0, 'c'},
static struct option long_options[] = {{"server", optional_argument, 0, 's'},
{"port", optional_argument, 0, 'p'},
{"help", no_argument, 0, 'h'},
{NULL, 0, 0, 0}};
......@@ -28,9 +29,9 @@ main(int argc, char *argv[]) {
app_name = argv[0];
int value;
while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) {
while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) {
switch (value) {
case 'h': {
case 's': {
char *address_ptr = strdup(optarg);
address = address_ptr;
free(address_ptr);
......@@ -38,12 +39,14 @@ main(int argc, char *argv[]) {
}
case 'p': {
char *port_ptr = strdup(optarg);
address = port_ptr;
port = port_ptr;
free(port_ptr);
break;
}
case 'h':
default:
break;
print_help(app_name);
return EXIT_SUCCESS;
}
}
......@@ -58,7 +61,8 @@ void
print_help(const std::string &app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -h Megasearch server address\n");
printf(" -p Megasearch server port\n");
printf(" -s --server Server address, default 127.0.0.1\n");
printf(" -p --port Server port, default 33001\n");
printf(" -h --help Print help information\n");
printf("\n");
}
\ No newline at end of file
cpp/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @ 4193b2e3
......@@ -13,13 +13,59 @@
using namespace megasearch;
namespace {
std::string GetTableName();
static const std::string TABLE_NAME = GetTableName();
static const std::string VECTOR_COLUMN_NAME = "face_vector";
static const std::string ID_COLUMN_NAME = "aid";
static const std::string CITY_COLUMN_NAME = "city";
static constexpr int64_t TABLE_DIMENSION = 512;
static constexpr int64_t TOTAL_ROW_COUNT = 100000;
static constexpr int64_t TOP_K = 10;
static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
void PrintTableSchema(const megasearch::TableSchema& tb_schema) {
std::cout << "===========================================" << std::endl;
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl;
std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl;
std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl;
std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl;
std::cout << "===========================================" << std::endl;
BLOCK_SPLITER
}
void PrintRecordIdArray(const std::vector<int64_t>& record_ids) {
BLOCK_SPLITER
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
#if 0
for(auto id : record_ids) {
std::cout << std::to_string(id) << std::endl;
}
#endif
BLOCK_SPLITER
}
void PrintSearchResult(const std::vector<TopKQueryResult>& topk_query_result_array) {
BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
int32_t index = 0;
for(auto& result : topk_query_result_array) {
index++;
std::cout << "No." << std::to_string(index) << " vector top "
<< std::to_string(result.query_result_arrays.size())
<< " search result:" << std::endl;
for(auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score);
for(auto& attribute : item.column_map) {
std::cout << "\t" << attribute.first << ":" << attribute.second;
}
std::cout << std::endl;
}
}
BLOCK_SPLITER
}
std::string CurrentTime() {
......@@ -40,9 +86,24 @@ namespace {
return s_id;
}
static const std::string TABLE_NAME = GetTableName();
static const std::string VECTOR_COLUMN_NAME = "face_vector";
static const int64_t TABLE_DIMENSION = 512;
TableSchema BuildTableSchema() {
TableSchema tb_schema;
VectorColumn col1;
col1.name = VECTOR_COLUMN_NAME;
col1.dimension = TABLE_DIMENSION;
col1.store_raw_vector = true;
tb_schema.vector_column_array.emplace_back(col1);
Column col2 = {ColumnType::int8, ID_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col2);
Column col3 = {ColumnType::int16, CITY_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col3);
tb_schema.table_name = TABLE_NAME;
return tb_schema;
}
void BuildVectors(int64_t from, int64_t to,
std::vector<RowRecord>* vector_record_array,
......@@ -58,6 +119,19 @@ namespace {
query_record_array->clear();
}
static const std::map<int64_t , std::string> CITY_MAP = {
{0, "Beijing"},
{1, "Shanhai"},
{2, "Hangzhou"},
{3, "Guangzhou"},
{4, "Shenzheng"},
{5, "Wuhan"},
{6, "Chengdu"},
{7, "Chongqin"},
{8, "Tianjing"},
{9, "Hongkong"},
};
for (int64_t k = from; k < to; k++) {
std::vector<float> f_p;
......@@ -69,12 +143,16 @@ namespace {
if(vector_record_array) {
RowRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.attribute_map[ID_COLUMN_NAME] = std::to_string(k);
record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size());
vector_record_array->emplace_back(record);
}
if(query_record_array) {
QueryRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.selected_column_array.push_back(ID_COLUMN_NAME);
record.selected_column_array.push_back(CITY_COLUMN_NAME);
query_record_array->emplace_back(record);
}
}
......@@ -84,61 +162,80 @@ namespace {
void
ClientTest::Test(const std::string& address, const std::string& port) {
std::shared_ptr<Connection> conn = Connection::Create();
ConnectParam param = { address, port };
conn->Connect(param);
{//create table
TableSchema tb_schema;
VectorColumn col1;
col1.name = VECTOR_COLUMN_NAME;
col1.dimension = TABLE_DIMENSION;
col1.store_raw_vector = true;
tb_schema.vector_column_array.emplace_back(col1);
{//connect server
ConnectParam param = {address, port};
Status stat = conn->Connect(param);
std::cout << "Connect function call status: " << stat.ToString() << std::endl;
}
Column col2;
col2.name = "age";
tb_schema.attribute_column_array.emplace_back(col2);
{//server version
std::string version = conn->ServerVersion();
std::cout << "MegaSearch server version: " << version << std::endl;
}
tb_schema.table_name = TABLE_NAME;
{//sdk version
std::string version = conn->ClientVersion();
std::cout << "SDK version: " << version << std::endl;
}
{
std::vector<std::string> tables;
Status stat = conn->ShowTables(tables);
std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
std::cout << "All tables: " << std::endl;
for(auto& table : tables) {
std::cout << "\t" << table << std::endl;
}
}
{//create table
TableSchema tb_schema = BuildTableSchema();
PrintTableSchema(tb_schema);
Status stat = conn->CreateTable(tb_schema);
std::cout << "Create table result: " << stat.ToString() << std::endl;
std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
}
{//describe table
TableSchema tb_schema;
Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
std::cout << "Describe table result: " << stat.ToString() << std::endl;
std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
PrintTableSchema(tb_schema);
}
{//add vectors
std::vector<RowRecord> record_array;
BuildVectors(0, 10000, &record_array, nullptr);
BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr);
std::vector<int64_t> record_ids;
std::cout << "Begin add vectors" << std::endl;
Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
std::cout << "Add vector result: " << stat.ToString() << std::endl;
std::cout << "Returned vector ids: " << record_ids.size() << std::endl;
std::cout << "AddVector function call status: " << stat.ToString() << std::endl;
PrintRecordIdArray(record_ids);
}
{//search vectors
std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl;
sleep(10);
std::vector<QueryRecord> record_array;
BuildVectors(500, 510, nullptr, &record_array);
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array);
std::vector<TopKQueryResult> topk_query_result_array;
std::cout << "Begin search vectors" << std::endl;
Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, 10);
std::cout << "Search vector result: " << stat.ToString() << std::endl;
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
PrintSearchResult(topk_query_result_array);
}
// {//delete table
// Status stat = conn->DeleteTable(TABLE_NAME);
// std::cout << "Delete table result: " << stat.ToString() << std::endl;
// std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
// }
{//server status
std::string status = conn->ServerStatus();
std::cout << "Server status before disconnect: " << status << std::endl;
}
Connection::Destroy(conn);
{//server status
std::string status = conn->ServerStatus();
std::cout << "Server status after disconnect: " << status << std::endl;
}
}
\ No newline at end of file
cpp/src/sdk/include/Status.h
浏览文件 @ 4193b2e3
......@@ -14,7 +14,8 @@ enum class StatusCode {
OK = 0,
Invalid = 1,
UnknownError = 2,
NotSupported = 3
NotSupported = 3,
NotConnected = 4
};
/**
......
cpp/src/sdk/src/client/ClientProxy.cpp
浏览文件 @ 4193b2e3
......@@ -16,32 +16,50 @@ ClientProxy::ClientPtr() const {
return client_ptr;
}
bool ClientProxy::IsConnected() const {
return (client_ptr != nullptr && connected_);
}
Status
ClientProxy::Connect(const ConnectParam &param) {
Disconnect();
int32_t port = atoi(param.port.c_str());
return ClientPtr()->Connect(param.ip_address, port, "json");
Status status = ClientPtr()->Connect(param.ip_address, port, THRIFT_PROTOCOL_BINARY);
if(status.ok()) {
connected_ = true;
}
return status;
}
Status
ClientProxy::Connect(const std::string &uri) {
Disconnect();
return Status::NotSupported("Connect interface is not supported.");
size_t index = uri.find_first_of(":", 0);
if((index == std::string::npos)) {
return Status::Invalid("Invalid uri");
}
ConnectParam param;
param.ip_address = uri.substr(0, index);
param.port = uri.substr(index + 1);
return Connect(param);
}
Status
ClientProxy::Connected() const {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
std::string info;
ClientPtr()->interface()->Ping(info, "");
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "connection lost: " + std::string(ex.what()));
return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
}
return Status::OK();
......@@ -49,22 +67,23 @@ ClientProxy::Connected() const {
Status
ClientProxy::Disconnect() {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
connected_ = false;
return ClientPtr()->Disconnect();
}
std::string
ClientProxy::ClientVersion() const {
return std::string("Current Version");
return std::string("v1.0");
}
Status
ClientProxy::CreateTable(const TableSchema &param) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -103,8 +122,8 @@ ClientProxy::CreateTable(const TableSchema &param) {
Status
ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -132,8 +151,8 @@ ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
Status
ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -152,8 +171,8 @@ ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
Status
ClientProxy::DeleteTable(const std::string &table_name) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -170,8 +189,8 @@ Status
ClientProxy::AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -205,8 +224,8 @@ ClientProxy::SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -253,8 +272,8 @@ ClientProxy::SearchVector(const std::string &table_name,
Status
ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -290,8 +309,8 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
Status
ClientProxy::ShowTables(std::vector<std::string> &table_array) {
if(client_ptr == nullptr) {
return Status(StatusCode::UnknownError, "not connected");
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
......@@ -306,7 +325,7 @@ ClientProxy::ShowTables(std::vector<std::string> &table_array) {
std::string
ClientProxy::ServerVersion() const {
if(client_ptr == nullptr) {
if(!IsConnected()) {
return "";
}
......@@ -321,8 +340,8 @@ ClientProxy::ServerVersion() const {
std::string
ClientProxy::ServerStatus() const {
if(client_ptr == nullptr) {
return "not connected";
if(!IsConnected()) {
return "not connected to server";
}
try {
......
cpp/src/sdk/src/client/ClientProxy.h
浏览文件 @ 4193b2e3
......@@ -51,9 +51,11 @@ public:
private:
std::shared_ptr<ThriftClient>& ClientPtr() const;
bool IsConnected() const;
private:
mutable std::shared_ptr<ThriftClient> client_ptr;
bool connected_ = false;
};
}
cpp/src/sdk/src/client/ThriftClient.cpp
浏览文件 @ 4193b2e3
......@@ -50,14 +50,12 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
stdcxx::shared_ptr<TSocket> socket_ptr(new transport::TSocket(address, port));
stdcxx::shared_ptr<TTransport> transport_ptr(new TBufferedTransport(socket_ptr));
stdcxx::shared_ptr<TProtocol> protocol_ptr;
if(protocol == "binary") {
if(protocol == THRIFT_PROTOCOL_BINARY) {
protocol_ptr.reset(new TBinaryProtocol(transport_ptr));
} else if(protocol == "json") {
} else if(protocol == THRIFT_PROTOCOL_JSON) {
protocol_ptr.reset(new TJSONProtocol(transport_ptr));
} else if(protocol == "compact") {
} else if(protocol == THRIFT_PROTOCOL_COMPACT) {
protocol_ptr.reset(new TCompactProtocol(transport_ptr));
} else if(protocol == "debug") {
protocol_ptr.reset(new TDebugProtocol(transport_ptr));
} else {
//CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently";
return Status(StatusCode::Invalid, "unsupported protocol");
......@@ -67,7 +65,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
client_ = std::make_shared<thrift::MegasearchServiceClient>(protocol_ptr);
} catch ( std::exception& ex) {
//CLIENT_LOG_ERROR << "connect encounter exception: " << ex.what();
return Status(StatusCode::UnknownError, "failed to connect megasearch server" + std::string(ex.what()));
return Status(StatusCode::NotConnected, "failed to connect megasearch server" + std::string(ex.what()));
}
return Status::OK();
......
cpp/src/sdk/src/client/ThriftClient.h
浏览文件 @ 4193b2e3
......@@ -14,6 +14,10 @@ namespace megasearch {
using MegasearchServiceClientPtr = std::shared_ptr<megasearch::thrift::MegasearchServiceClient>;
static const std::string THRIFT_PROTOCOL_JSON = "json";
static const std::string THRIFT_PROTOCOL_BINARY = "binary";
static const std::string THRIFT_PROTOCOL_COMPACT = "compact";
class ThriftClient {
public:
ThriftClient();
......
cpp/src/sdk/src/interface/Status.cpp
浏览文件 @ 4193b2e3
......@@ -101,6 +101,8 @@ std::string Status::CodeAsString() const {
break;
case StatusCode::NotSupported: type = "Not Supported";
break;
case StatusCode::NotConnected: type = "Not Connected";
break;
default: type = "Unknown";
break;
}
......
cpp/src/server/MegasearchHandler.cpp
浏览文件 @ 4193b2e3
......@@ -32,14 +32,14 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) {
void
MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam &param) {
// Your implementation goes here
printf("CreateTablePartition\n");
BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam &param) {
// Your implementation goes here
printf("DeleteTablePartition\n");
BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
......@@ -67,14 +67,14 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std:
void
MegasearchServiceHandler::ShowTables(std::vector<std::string> &_return) {
// Your implementation goes here
printf("ShowTables\n");
BaseTaskPtr task_ptr = ShowTablesTask::Create(_return);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::Ping(std::string& _return, const std::string& cmd) {
// Your implementation goes here
printf("Ping\n");
BaseTaskPtr task_ptr = PingTask::Create(cmd, _return);
MegasearchScheduler::ExecTask(task_ptr);
}
}
......
cpp/src/server/MegasearchServer.cpp
浏览文件 @ 4193b2e3
......@@ -54,7 +54,6 @@ MegasearchServer::StartService() {
stdcxx::shared_ptr<TServerTransport> server_transport(new TServerSocket(address, port));
stdcxx::shared_ptr<TTransportFactory> transport_factory(new TBufferedTransportFactory());
std::string protocol = "json";
stdcxx::shared_ptr<TProtocolFactory> protocol_factory;
if (protocol == "binary") {
protocol_factory.reset(new TBinaryProtocolFactory());
......@@ -62,8 +61,6 @@ MegasearchServer::StartService() {
protocol_factory.reset(new TJSONProtocolFactory());
} else if (protocol == "compact") {
protocol_factory.reset(new TCompactProtocolFactory());
} else if (protocol == "debug") {
protocol_factory.reset(new TDebugProtocolFactory());
} else {
//SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently";
return;
......
cpp/src/server/MegasearchTask.cpp
浏览文件 @ 4193b2e3
......@@ -21,6 +21,7 @@ namespace server {
static const std::string DQL_TASK_GROUP = "dql";
static const std::string DDL_DML_TASK_GROUP = "ddl_dml";
static const std::string PING_TASK_GROUP = "ping";
static const std::string VECTOR_UID = "uid";
static const uint64_t USE_MT = 5000;
......@@ -48,6 +49,10 @@ namespace {
}
}
~DBWrapper() {
delete db_;
}
zilliz::vecwise::engine::DB* DB() { return db_; }
private:
......@@ -78,17 +83,17 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) {
ServerError CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask");
try {
if(schema_.vector_column_array.empty()) {
return SERVER_INVALID_ARGUMENT;
}
IVecIdMapper::GetInstance()->AddGroup(schema_.table_name);
engine::meta::TableSchema table_schema;
table_schema.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
table_schema.table_id = schema_.table_name;
engine::Status stat = DB()->CreateTable(table_schema);
engine::meta::TableSchema table_info;
table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
table_info.table_id = schema_.table_name;
engine::Status stat = DB()->CreateTable(table_info);
if(!stat.ok()) {//could exist
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
......@@ -109,7 +114,7 @@ ServerError CreateTableTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DescribeTableTask::DescribeTableTask(const std::string &table_name, thrift::TableSchema &schema)
: BaseTask(DDL_DML_TASK_GROUP),
: BaseTask(PING_TASK_GROUP),
table_name_(table_name),
schema_(schema) {
schema_.table_name = table_name_;
......@@ -123,9 +128,9 @@ ServerError DescribeTableTask::OnExecute() {
TimeRecorder rc("DescribeTableTask");
try {
engine::meta::TableSchema table_schema;
table_schema.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_schema);
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
......@@ -154,8 +159,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name)
}
BaseTaskPtr DeleteTableTask::Create(const std::string& table_id) {
return std::shared_ptr<BaseTask>(new DeleteTableTask(table_id));
BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) {
return std::shared_ptr<BaseTask>(new DeleteTableTask(group_id));
}
ServerError DeleteTableTask::OnExecute() {
......@@ -168,6 +173,60 @@ ServerError DeleteTableTask::OnExecute() {
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new CreateTablePartitionTask(param));
}
ServerError CreateTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "create table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new DeleteTablePartitionTask(param));
}
ServerError DeleteTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "delete table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ShowTablesTask::ShowTablesTask(std::vector<std::string>& tables)
: BaseTask(PING_TASK_GROUP),
tables_(tables) {
}
BaseTaskPtr ShowTablesTask::Create(std::vector<std::string>& tables) {
return std::shared_ptr<BaseTask>(new ShowTablesTask(tables));
}
ServerError ShowTablesTask::OnExecute() {
IVecIdMapper::GetInstance()->AllGroups(tables_);
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
AddVectorTask::AddVectorTask(const std::string& table_name,
......@@ -178,7 +237,6 @@ AddVectorTask::AddVectorTask(const std::string& table_name,
record_array_(record_array),
record_ids_(record_ids) {
record_ids_.clear();
record_ids_.resize(record_array.size());
}
BaseTaskPtr AddVectorTask::Create(const std::string& table_name,
......@@ -195,9 +253,9 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
engine::meta::TableSchema table_schema;
table_schema.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_schema);
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
......@@ -208,7 +266,7 @@ ServerError AddVectorTask::OnExecute() {
rc.Record("get group info");
uint64_t vec_count = (uint64_t)record_array_.size();
uint64_t group_dim = table_schema.dimension;
uint64_t group_dim = table_info.dimension;
std::vector<float> vec_f;
vec_f.resize(vec_count*group_dim);//allocate enough memory
for(uint64_t i = 0; i < vec_count; i++) {
......@@ -228,6 +286,7 @@ ServerError AddVectorTask::OnExecute() {
return error_code_;
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
......@@ -245,12 +304,27 @@ ServerError AddVectorTask::OnExecute() {
return error_code_;
}
if(record_ids_.size() < vec_count) {
if(record_ids_.size() != vec_count) {
SERVER_LOG_ERROR << "Vector ID not returned";
return SERVER_UNEXPECTED_ERROR;
}
rc.Record("done");
//persist attributes
for(uint64_t i = 0; i < vec_count; i++) {
const auto &record = record_array_[i];
//any attributes?
if(record.attribute_map.empty()) {
continue;
}
std::string nid = std::to_string(record_ids_[i]);
std::string attrib_str;
AttributeSerializer::Encode(record.attribute_map, attrib_str);
IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_);
}
rc.Record("persist vector attributes");
} catch (std::exception& ex) {
error_code_ = SERVER_UNEXPECTED_ERROR;
......@@ -293,9 +367,9 @@ ServerError SearchVectorTask::OnExecute() {
return error_code_;
}
engine::meta::TableSchema table_schema;
table_schema.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_schema);
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
......@@ -305,7 +379,7 @@ ServerError SearchVectorTask::OnExecute() {
std::vector<float> vec_f;
uint64_t record_count = (uint64_t)record_array_.size();
vec_f.resize(record_count*table_schema.dimension);
vec_f.resize(record_count*table_info.dimension);
for(uint64_t i = 0; i < record_array_.size(); i++) {
const auto& record = record_array_[i];
......@@ -317,14 +391,15 @@ ServerError SearchVectorTask::OnExecute() {
}
uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value?
if (vec_dim != table_schema.dimension) {
if (vec_dim != table_info.dimension) {
SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
<< " vs. group dimension:" << table_schema.dimension;
<< " vs. group dimension:" << table_info.dimension;
error_code_ = SERVER_INVALID_VECTOR_DIMENSION;
error_msg_ = "Engine failed: " + stat.ToString();
return error_code_;
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
......@@ -336,25 +411,50 @@ ServerError SearchVectorTask::OnExecute() {
std::vector<DB_DATE> dates;
engine::QueryResults results;
stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results);
rc.Record("search vectors from engine");
if(!stat.ok()) {
SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
return SERVER_UNEXPECTED_ERROR;
} else {
rc.Record("do searching");
for(engine::QueryResult& result : results){
thrift::TopKQueryResult thrift_topk_result;
for(auto id : result) {
thrift::QueryResult thrift_result;
thrift_result.__set_id(id);
}
if(results.size() != record_count) {
SERVER_LOG_ERROR << "Search result not returned";
return SERVER_UNEXPECTED_ERROR;
}
//construct result array
for(uint64_t i = 0; i < record_count; i++) {
auto& result = results[i];
const auto& record = record_array_[i];
thrift::TopKQueryResult thrift_topk_result;
for(auto id : result) {
thrift::QueryResult thrift_result;
thrift_result.__set_id(id);
//need get attributes?
if(record.selected_column_array.empty()) {
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
continue;
}
result_array_.emplace_back(thrift_topk_result);
std::string nid = std::to_string(id);
std::string attrib_str;
IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_);
AttribMap attrib_map;
AttributeSerializer::Decode(attrib_str, attrib_map);
for(auto& attribute : record.selected_column_array) {
thrift_result.column_map[attribute] = attrib_map[attribute];
}
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
}
rc.Record("construct result");
}
rc.Record("done");
result_array_.emplace_back(thrift_topk_result);
}
rc.Record("construct result");
} catch (std::exception& ex) {
error_code_ = SERVER_UNEXPECTED_ERROR;
......@@ -366,6 +466,26 @@ ServerError SearchVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PingTask::PingTask(const std::string& cmd, std::string& result)
: BaseTask(PING_TASK_GROUP),
cmd_(cmd),
result_(result) {
}
BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
return std::shared_ptr<BaseTask>(new PingTask(cmd, result));
}
ServerError PingTask::OnExecute() {
if(cmd_ == "version") {
result_ = "v1.2.0";//currently hardcode
}
return SERVER_SUCCESS;
}
}
}
}
cpp/src/server/MegasearchTask.h
浏览文件 @ 4193b2e3
......@@ -65,6 +65,50 @@ private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::CreateTablePartitionParam &param);
protected:
CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::CreateTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DeleteTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam &param);
protected:
DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::DeleteTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowTablesTask : public BaseTask {
public:
static BaseTaskPtr Create(std::vector<std::string>& tables);
protected:
ShowTablesTask(std::vector<std::string>& tables);
ServerError OnExecute() override;
private:
std::vector<std::string>& tables_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class AddVectorTask : public BaseTask {
public:
......@@ -108,6 +152,21 @@ private:
std::vector<thrift::TopKQueryResult>& result_array_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class PingTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& cmd, std::string& result);
protected:
PingTask(const std::string& cmd, std::string& result);
ServerError OnExecute() override;
private:
std::string cmd_;
std::string& result_;
};
}
}
}
\ No newline at end of file
cpp/src/server/RocksIdMapper.cpp
浏览文件 @ 4193b2e3
......@@ -9,6 +9,7 @@
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
......@@ -108,6 +109,19 @@ bool RocksIdMapper::IsGroupExist(const std::string& group) const {
return IsGroupExistInternal(group);
}
ServerError RocksIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
std::lock_guard<std::mutex> lck(db_mutex_);
for(auto& pair : column_handles_) {
if(pair.first == ROCKSDB_DEFAULT_GROUP) {
continue;
}
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
......
cpp/src/server/RocksIdMapper.h
浏览文件 @ 4193b2e3
......@@ -8,13 +8,16 @@
#include "utils/Error.h"
#include "VecIdMapper.h"
#include "rocksdb/db.h"
#include <string>
#include <vector>
#include <unordered_map>
#include <mutex>
namespace rocksdb {
class DB;
class ColumnFamilyHandle;
}
namespace zilliz {
namespace vecwise {
namespace server {
......@@ -26,6 +29,7 @@ class RocksIdMapper : public IVecIdMapper{
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
......
cpp/src/server/ServerConfig.h
浏览文件 @ 4193b2e3
......@@ -8,7 +8,7 @@
#include "utils/Error.h"
#include "config/ConfigNode.h"
#include <yaml-cpp/yaml.h>
#include "yaml-cpp/yaml.h"
namespace zilliz {
namespace vecwise {
......
cpp/src/server/VecIdMapper.cpp
浏览文件 @ 4193b2e3
......@@ -15,6 +15,7 @@
#include "rocksdb/options.h"
#include <exception>
#include <unordered_map>
namespace zilliz {
namespace vecwise {
......@@ -30,6 +31,30 @@ IVecIdMapper* IVecIdMapper::GetInstance() {
#endif
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class SimpleIdMapper : public IVecIdMapper{
public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SimpleIdMapper::SimpleIdMapper() {
......@@ -52,6 +77,15 @@ SimpleIdMapper::IsGroupExist(const std::string& group) const {
return id_groups_.count(group) > 0;
}
ServerError SimpleIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
for(auto& pair : id_groups_) {
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
......
cpp/src/server/VecIdMapper.h
浏览文件 @ 4193b2e3
......@@ -9,11 +9,6 @@
#include <string>
#include <vector>
#include <unordered_map>
namespace rocksdb {
class DB;
}
namespace zilliz {
namespace vecwise {
......@@ -27,6 +22,7 @@ public:
virtual ServerError AddGroup(const std::string& group) = 0;
virtual bool IsGroupExist(const std::string& group) const = 0;
virtual ServerError AllGroups(std::vector<std::string>& groups) const = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
......@@ -39,28 +35,6 @@ public:
virtual ServerError DeleteGroup(const std::string& group) = 0;
};
class SimpleIdMapper : public IVecIdMapper{
public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
}
}
}
cpp/thirdparty/versions.txt 0 → 100644
浏览文件 @ 4193b2e3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Toolchain library versions
#
# This file is used by `download_dependencies.sh` and cmake to figure out which
# version of a dependency to fetch. In order to add a new dependency, add a
# version variable, e.g. MY_DEP_VERSION and append an entry in the
# `DEPENDENCIES` array (see the comment on top of the declaration for the
# format).
BOOST_VERSION=1.70.0
BZIP2_VERSION=1.0.6
EASYLOGGINGPP_VERSION=v9.96.7
FAISS_VERSION=v1.5.2
GTEST_VERSION=1.8.1
LAPACK_VERSION=v3.8.0
LZ4_VERSION=v1.9.1
OPENBLAS_VERSION=v0.3.6
PROMETHEUS_VERSION=master
ROCKSDB_VERSION=v6.0.2
SNAPPY_VERSION=1.1.7
SQLITE_VERSION=3280000
SQLITE_ORM_VERSION=master
THRIFT_VERSION=0.12.0
YAMLCPP_VERSION=0.6.2
ZLIB_VERSION=v1.2.11
ZSTD_VERSION=v1.4.0
# The first field is the name of the environment variable expected by cmake.
# This _must_ match what is defined. The second field is the name of the
# generated archive file. The third field is the url of the project for the
# given version.
DEPENDENCIES=(
"MEGASEARCH_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz"
"MEGASEARCH_BZIP2_URL bzip2-${BZIP2_VERSION}.tar.gz https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz"
"MEGASEARCH_EASYLOGGINGPP_URL easyloggingpp-${EASYLOGGINGPP_VERSION}.tar.gz https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz"
"MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz"
"MEGASEARCH_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
"MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz
"MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz"
"MEGASEARCH_OPENBLAS_URL openblas-${OPENBLAS_VERSION}.tar.gz https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz"
"MEGASEARCH_PROMETHEUS_URL prometheus-${PROMETHEUS_VERSION}.tar.gz https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz"
"MEGASEARCH_ROCKSDB_URL rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz"
"MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz"
"MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz"
"MEGASEARCH_SQLITE_ORM_URL sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz"
"MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
"MEGASEARCH_YAMLCPP_URL yaml-cpp-${YAMLCPP_VERSION}.tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz"
"MEGASEARCH_ZLIB_URL zlib-${ZLIB_VERSION}.tar.gz https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz"
"MEGASEARCH_ZSTD_URL zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz"
)
# vim: set filetype=sh:
\ No newline at end of file
cpp/unittest/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -5,18 +5,26 @@
#-------------------------------------------------------------------------------
link_directories(
"${CMAKE_BINARY_DIR}/lib"
"${VECWISE_THIRD_PARTY_BUILD}/lib"
#"${VECWISE_THIRD_PARTY_BUILD}/lib"
"${GTEST_PREFIX}/lib/"
)
message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib")
set(unittest_srcs
${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc)
${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp)
#${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc)
set(unittest_libs
yaml-cpp
gtest
gmock
gtest_main
gmock_main
easyloggingpp
pthread
metrics)
metrics
openblas
gfortran)
add_subdirectory(server)
add_subdirectory(db)
......
cpp/unittest/db/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -42,4 +42,4 @@ set(db_libs
lz4
)
target_link_libraries(db_test ${unittest_libs} ${db_libs})
target_link_libraries(db_test ${db_libs} ${unittest_libs})
cpp/unittest/faiss_wrapper/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -45,7 +45,7 @@ set(wrapper_libs
zstd
lz4
)
target_link_libraries(wrapper_test ${unittest_libs} ${wrapper_libs})
target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs})
set(topk_test_src
topk_test.cpp
......
cpp/unittest/license/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -16,6 +16,7 @@ link_directories(/usr/local/cuda/lib64)
link_directories(/usr/lib/x86_64-linux-gnu)
link_directories(/usr/lib/nvidia-415)
link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/)
link_directories(/usr/local/cuda/lib64/stubs/)
set(require_files
../../src/license/LicenseLibrary.cpp
......@@ -44,4 +45,4 @@ set(db_libs
boost_serialization
)
target_link_libraries(license_test ${unittest_libs} ${db_libs})
target_link_libraries(license_test ${db_libs} ${unittest_libs})
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -57,7 +57,7 @@ set(count_test_src
add_executable(metrics_test ${count_test_src} ${require_files} )
target_link_libraries(metrics_test
${unittest_libs}
faiss
cudart
cublas
......@@ -72,5 +72,5 @@ target_link_libraries(metrics_test
# prometheus-cpp-core
pthread
z
${unittest_libs}
)
\ No newline at end of file
cpp/unittest/server/CMakeLists.txt
浏览文件 @ 4193b2e3
......@@ -35,23 +35,23 @@ cuda_add_executable(server_test
set(require_libs
stdc++
boost_system
boost_filesystem
pthread
snappy
bz2
z
zstd
rocksdb
faiss
cudart
cublas
sqlite3
boost_system
boost_filesystem
snappy
z
bz2
zstd
lz4
pthread
)
target_link_libraries(server_test
${unittest_libs}
${require_libs}
${cuda_library}
${unittest_libs}
)
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或