diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index af2151c5a5dd5ff6a032d5d9e904c0204ee3f60a..efb810e2a58a2041a6ead6f57d7a72545533ae47 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -12,9 +12,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-5 - Implement Auto Archive Feature - MS-16 - Implement metrics without prometheus +- MS-6 - Implement SDK interface part 1 +- MS-21 - Implement SDK interface part 2 +- MS-26 - cmake. Add thirdparty packages +- MS-31 - cmake: add prometheus + ### Task - MS-1 - Add CHANGELOG.md - MS-4 - Refactor the vecwise_engine code structure -- MS-6 - Implement SDK interface part 1 - MS-20 - Clean Code Part 1 +- MS-30 - Use faiss v1.5.2 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3312a128c40dbe9fcec9d4e8a049810f5c3a7a64..f4e32948dd72136eecb44f706c2eb857c2ff2300 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -4,10 +4,33 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.14) +message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(MEGASEARCH_VERSION "0.1.0") + +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}") + +project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}") project(vecwise_engine LANGUAGES CUDA CXX) +set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}") +set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}") +set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}") + +if(MEGASEARCH_VERSION_MAJOR STREQUAL "" + OR MEGASEARCH_VERSION_MINOR STREQUAL "" + OR MEGASEARCH_VERSION_PATCH STREQUAL "") + message(FATAL_ERROR "Failed to determine MegaSearch version from '${MEGASEARCH_VERSION}'") +endif() + +message(STATUS "MegaSearch version: " + "${MEGASEARCH_VERSION_MAJOR}.${MEGASEARCH_VERSION_MINOR}.${MEGASEARCH_VERSION_PATCH} " + "(full: '${MEGASEARCH_VERSION}')") + +set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR}) + find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") @@ -19,6 +42,7 @@ if (GPU_VERSION STREQUAL "ON") endif () set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") message("building vecwise_engine on x86 architecture") @@ -32,49 +56,52 @@ else() set(VECWISE_BUILD_ARCH unknown) endif() -if(DEFINED UNIX) - message("building vecwise on Unix") - set(VECWISE_BUILD_SYSTEM macos) -elseif(DEFINED APPLE) - message("building vecwise on MacOS") - set(VECWISE_BUILD_SYSTEM unix) -else() - message("unknown OS") - set(VECWISE_BUILD_SYSTEM unknown) -endif () - if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") if (GPU_VERSION STREQUAL "ON") set(ENABLE_LICENSE "ON") add_definitions("-DENABLE_LICENSE") endif () else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH}) + + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +#if (BUILD_UNIT_TEST) +# option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON) +#endif(BUILD_UNIT_TEST) + +include(ExternalProject) +include(DefineOptions) +include(BuildUtils) +include(ThirdPartyPackages) + +include_directories(${MEGASEARCH_SOURCE_DIR}) +link_directories(${MEGASEARCH_BINARY_DIR}) + +## Following should be check set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) -set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) -set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) +#set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) +#set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) add_compile_definitions(PROFILER=${PROFILER}) include_directories(${VECWISE_ENGINE_INCLUDE}) include_directories(${VECWISE_ENGINE_SRC}) -include_directories(${VECWISE_THIRD_PARTY_BUILD}/include) +#include_directories(${VECWISE_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib) -link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64) +#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib) +#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64) #execute_process(COMMAND bash build.sh # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) add_subdirectory(src) -add_subdirectory(unittest) - if (BUILD_UNIT_TEST) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) @@ -93,3 +120,5 @@ install(FILES conf/vecwise_engine_log.conf DESTINATION conf) + +config_summary() diff --git a/cpp/build.sh b/cpp/build.sh index fde83360ba6443ef5b166f98b46231fa43867906..abab7bdffafcad4188cded1429aad01aeaa786b4 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -72,7 +72,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then make clean fi -make -j || exit 1 +make -j 1 || exit 1 if [[ ${BUILD_TYPE} != "Debug" ]]; then strip src/vecwise_server diff --git a/cpp/cmake/BuildUtils.cmake b/cpp/cmake/BuildUtils.cmake new file mode 100644 index 0000000000000000000000000000000000000000..361d038c2277320f46551b5759358782da0b50e4 --- /dev/null +++ b/cpp/cmake/BuildUtils.cmake @@ -0,0 +1,95 @@ +function(ADD_THIRDPARTY_LIB LIB_NAME) + set(options) + set(one_value_args SHARED_LIB STATIC_LIB) + set(multi_value_args DEPS INCLUDE_DIRECTORIES) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + if(ARG_STATIC_LIB AND ARG_SHARED_LIB) + if(NOT ARG_STATIC_LIB) + message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") + endif() + + set(AUG_LIB_NAME "${LIB_NAME}_static") + add_library(${AUG_LIB_NAME} STATIC IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + + set(AUG_LIB_NAME "${LIB_NAME}_shared") + add_library(${AUG_LIB_NAME} SHARED IMPORTED) + + if(WIN32) + # Mark the ".lib" location as part of a Windows DLL + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") + else() + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") + endif() + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + elseif(ARG_STATIC_LIB) + set(AUG_LIB_NAME "${LIB_NAME}_static") + add_library(${AUG_LIB_NAME} STATIC IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + elseif(ARG_SHARED_LIB) + set(AUG_LIB_NAME "${LIB_NAME}_shared") + add_library(${AUG_LIB_NAME} SHARED IMPORTED) + + if(WIN32) + # Mark the ".lib" location as part of a Windows DLL + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") + else() + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") + endif() + message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + else() + message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") + endif() +endfunction() \ No newline at end of file diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake new file mode 100644 index 0000000000000000000000000000000000000000..ce2e4ae6be4365a69a1e817b8f77987541aeefcc --- /dev/null +++ b/cpp/cmake/DefineOptions.cmake @@ -0,0 +1,196 @@ + +macro(set_option_category name) + set(MEGASEARCH_OPTION_CATEGORY ${name}) + list(APPEND "MEGASEARCH_OPTION_CATEGORIES" ${name}) +endmacro() + +macro(define_option name description default) + option(${name} ${description} ${default}) + list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name}) + set("${name}_OPTION_DESCRIPTION" ${description}) + set("${name}_OPTION_DEFAULT" ${default}) + set("${name}_OPTION_TYPE" "bool") +endmacro() + +function(list_join lst glue out) + if("${${lst}}" STREQUAL "") + set(${out} "" PARENT_SCOPE) + return() + endif() + + list(GET ${lst} 0 joined) + list(REMOVE_AT ${lst} 0) + foreach(item ${${lst}}) + set(joined "${joined}${glue}${item}") + endforeach() + set(${out} ${joined} PARENT_SCOPE) +endfunction() + +macro(define_option_string name description default) + set(${name} ${default} CACHE STRING ${description}) + list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name}) + set("${name}_OPTION_DESCRIPTION" ${description}) + set("${name}_OPTION_DEFAULT" "\"${default}\"") + set("${name}_OPTION_TYPE" "string") + + set("${name}_OPTION_ENUM" ${ARGN}) + list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM") + if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + set_property(CACHE ${name} PROPERTY STRINGS ${ARGN}) + endif() +endmacro() + +#---------------------------------------------------------------------- +set_option_category("Thirdparty") + +set(MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT "AUTO") + +define_option_string(MEGASEARCH_DEPENDENCY_SOURCE + "Method to use for acquiring MEGASEARCH's build dependencies" + "${MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM") + +define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD + "Show output from ExternalProjects rather than just logging to files" ON) + +define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) + +define_option(MEGASEARCH_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ +Note that this requires linking Boost statically" ON) + +define_option(MEGASEARCH_BOOST_HEADER_ONLY "Use only BOOST headers" OFF) + +define_option(MEGASEARCH_WITH_BZ2 "Build with BZ2 compression" ON) + +define_option(MEGASEARCH_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) + +define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON) + +define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON) + +define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against" + "-gencode=arch=compute_61,code=sm_61") + +define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON) + +define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON) + +define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON) + +define_option(MEGASEARCH_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) + +define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON) + +define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON) + +define_option(MEGASEARCH_WITH_SQLITE "Build with SQLite library" ON) + +define_option(MEGASEARCH_WITH_SQLITE_ORM "Build with SQLite ORM library" ON) + +define_option(MEGASEARCH_WITH_THRIFT "Build with Apache Thrift library" ON) + +define_option(MEGASEARCH_WITH_YAMLCPP "Build with yaml-cpp library" ON) + +define_option(MEGASEARCH_WITH_ZLIB "Build with zlib compression" ON) + +if(CMAKE_VERSION VERSION_LESS 3.7) + set(MEGASEARCH_WITH_ZSTD_DEFAULT OFF) +else() + # ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7. + set(MEGASEARCH_WITH_ZSTD_DEFAULT ON) +endif() +define_option(MEGASEARCH_WITH_ZSTD "Build with zstd compression" ${MEGASEARCH_WITH_ZSTD_DEFAULT}) + +#---------------------------------------------------------------------- +if(MSVC) + set_option_category("MSVC") + + define_option(MSVC_LINK_VERBOSE + "Pass verbose linking options when linking libraries and executables" + OFF) + + define_option(MEGASEARCH_USE_STATIC_CRT "Build MEGASEARCH with statically linked CRT" OFF) +endif() + + +#---------------------------------------------------------------------- +set_option_category("Test and benchmark") + +if (BUILD_UNIT_TEST) + define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON) +else() + define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" OFF) +endif(BUILD_UNIT_TEST) + +#---------------------------------------------------------------------- +macro(config_summary) + message(STATUS "---------------------------------------------------------------------") + message(STATUS "MEGASEARCH version: ${MEGASEARCH_VERSION}") + message(STATUS) + message(STATUS "Build configuration summary:") + + message(STATUS " Generator: ${CMAKE_GENERATOR}") + message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") + message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") + if(${CMAKE_EXPORT_COMPILE_COMMANDS}) + message( + STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") + endif() + + foreach(category ${MEGASEARCH_OPTION_CATEGORIES}) + + message(STATUS) + message(STATUS "${category} options:") + + set(option_names ${MEGASEARCH_${category}_OPTION_NAMES}) + + set(max_value_length 0) + foreach(name ${option_names}) + string(LENGTH "\"${${name}}\"" value_length) + if(${max_value_length} LESS ${value_length}) + set(max_value_length ${value_length}) + endif() + endforeach() + + foreach(name ${option_names}) + if("${${name}_OPTION_TYPE}" STREQUAL "string") + set(value "\"${${name}}\"") + else() + set(value "${${name}}") + endif() + + set(default ${${name}_OPTION_DEFAULT}) + set(description ${${name}_OPTION_DESCRIPTION}) + string(LENGTH ${description} description_length) + if(${description_length} LESS 70) + string( + SUBSTRING + " " + ${description_length} -1 description_padding) + else() + set(description_padding " + ") + endif() + + set(comment "[${name}]") + + if("${value}" STREQUAL "${default}") + set(comment "[default] ${comment}") + endif() + + if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + set(comment "${comment} [${${name}_OPTION_ENUM}]") + endif() + + string( + SUBSTRING "${value} " + 0 ${max_value_length} value) + + message(STATUS " ${description} ${description_padding} ${value} ${comment}") + endforeach() + + endforeach() + +endmacro() diff --git a/cpp/cmake/GoogleTest.cmake b/cpp/cmake/GoogleTest.cmake deleted file mode 100644 index 9ea62af1c4bec0cb5c96706d2b7a886ad7e7dc97..0000000000000000000000000000000000000000 --- a/cpp/cmake/GoogleTest.cmake +++ /dev/null @@ -1,92 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find GTest headers and libraries. -# -# Usage of this module as follows: -# -# find_package(GTest) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# GTest_HOME - When set, this path is inspected instead of standard library -# locations as the root of the GTest installation. -# The environment variable GTEST_HOME overrides this veriable. -# -# This module defines -# GTEST_INCLUDE_DIR, directory containing headers -# GTEST_LIBS, directory containing gtest libraries -# GTEST_STATIC_LIB, path to libgtest.a -# GTEST_SHARED_LIB, path to libgtest's shared library -# GTEST_FOUND, whether gtest has been found - -if( NOT "${GTEST_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path ) - list( APPEND _gtest_roots ${_native_path} ) -elseif ( GTest_HOME ) - list( APPEND _gtest_roots ${GTest_HOME} ) -endif() - -# Try the parameterized roots, if they exist -if ( _gtest_roots ) - find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( GTEST_LIBRARIES NAMES gtest gtest_main - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) -else () - find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h ) - find_library( GTEST_LIBRARIES NAMES gtest ) -endif () - - -if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) - set(GTEST_FOUND TRUE) - get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH ) - set(GTEST_LIB_NAME gtest) - set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(GTEST_FOUND FALSE) -endif () - -if (GTEST_FOUND) - if (NOT GTest_FIND_QUIETLY) - message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}") - endif () -else () - if (NOT GTest_FIND_QUIETLY) - set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ") - if ( _gtest_roots ) - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.") - else () - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.") - endif () - if (GTest_FIND_REQUIRED) - message(FATAL_ERROR "${GTEST_ERR_MSG}") - else (GTest_FIND_REQUIRED) - message(STATUS "${GTEST_ERR_MSG}") - endif (GTest_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - GTEST_INCLUDE_DIR - GTEST_LIBS - GTEST_LIBRARIES - GTEST_STATIC_LIB - GTEST_SHARED_LIB -) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index e85025a427dddbcc232fb2222f71cba211bbedda..0b08cd358a9d8a6c396422295773e0518edf954b 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -15,52 +15,1526 @@ # specific language governing permissions and limitations # under the License. -set(GTEST_VERSION "1.8.0") +set(MEGASEARCH_THIRDPARTY_DEPENDENCIES -message(MEGASEARCH_BUILD_TESTS ${MEGASEARCH_BUILD_TESTS}) + BOOST + BZip2 + Easylogging++ + FAISS + GTest + LAPACK + Lz4 + OpenBLAS + Prometheus + RocksDB + Snappy + SQLite + SQLite_ORM + Thrift + yaml-cpp + ZLIB + ZSTD) -if(MEGASEARCH_BUILD_TESTS) - add_custom_target(unittest ctest -L unittest) +message(STATUS "Using ${MEGASEARCH_DEPENDENCY_SOURCE} approach to find dependencies") - if("$ENV{GTEST_HOME}" STREQUAL "") - message("Yes") - set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +# For each dependency, set dependency source to global default, if unset +foreach(DEPENDENCY ${MEGASEARCH_THIRDPARTY_DEPENDENCIES}) + if("${${DEPENDENCY}_SOURCE}" STREQUAL "") + set(${DEPENDENCY}_SOURCE ${MEGASEARCH_DEPENDENCY_SOURCE}) + endif() +endforeach() + +macro(build_dependency DEPENDENCY_NAME) + if("${DEPENDENCY_NAME}" STREQUAL "BZip2") + build_bzip2() + elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++") + build_easyloggingpp() + elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") + build_faiss() + elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") + build_lapack() + elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") + build_lz4() + elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") + build_gtest() + elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") + build_openblas() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") + build_prometheus() + elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB") + build_rocksdb() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") + build_snappy() + elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") + build_sqlite() + elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM") + build_sqlite_orm() + elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift") + build_thrift() + elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") + build_yamlcpp() + elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") + build_zlib() + elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") + build_zstd() + else() + message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") + endif () +endmacro() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest") - set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GTEST_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_MAIN_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") +macro(resolve_dependency DEPENDENCY_NAME) + if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + #message(STATUS "Finding ${DEPENDENCY_NAME} package") +# find_package(${DEPENDENCY_NAME} QUIET) +# if (NOT ${DEPENDENCY_NAME}_FOUND) + #message(STATUS "${DEPENDENCY_NAME} package not found") + build_dependency(${DEPENDENCY_NAME}) +# endif () + elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") + build_dependency(${DEPENDENCY_NAME}) + elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + find_package(${DEPENDENCY_NAME} REQUIRED) + endif () +endmacro() - set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} - -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) +# ---------------------------------------------------------------------- +# ExternalProject options - ExternalProject_Add(googletest - URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" - BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" "${GTEST_MAIN_STATIC_LIB}" - CMAKE_ARGS ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - set(GTEST_VENDORED 1) +string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) + +set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") +set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") + +if(NOT MSVC) + # Set -fPIC on all external projects + set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") + set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") +endif() + +# CC/CXX environment variables are captured on the first invocation of the +# builder (e.g make or ninja) instead of when CMake is invoked into to build +# directory. This leads to issues if the variables are exported in a subshell +# and the invocation of make/ninja is in distinct subshell without the same +# environment (CC/CXX). +set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) + +if(CMAKE_AR) + set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) +endif() + +if(CMAKE_RANLIB) + set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB}) +endif() + +# External projects are still able to override the following declarations. +# cmake command line will favor the last defined variable when a duplicate is +# encountered. This requires that `EP_COMMON_CMAKE_ARGS` is always the first +# argument. +set(EP_COMMON_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) + +if(NOT MEGASEARCH_VERBOSE_THIRDPARTY_BUILD) + set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) +else() + set(EP_LOG_OPTIONS) +endif() + +# Ensure that a default make is set +if("${MAKE}" STREQUAL "") + if(NOT MSVC) + find_program(MAKE make) + endif() +endif() + +# Using make -j in sub-make is fragile +# see discussion https://github.com/apache/MEGASEARCH/pull/2779 +if(${CMAKE_GENERATOR} MATCHES "Makefiles") + set(MAKE_BUILD_ARGS "") +else() + # limit the maximum number of jobs for ninja + set(MAKE_BUILD_ARGS "-j4") +endif() + +# ---------------------------------------------------------------------- +# Find pthreads + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +# ---------------------------------------------------------------------- +# Versions and URLs for toolchain builds, which also can be used to configure +# offline builds + +# Read toolchain versions from cpp/thirdparty/versions.txt +set(THIRDPARTY_DIR "${MEGASEARCH_SOURCE_DIR}/thirdparty") +file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) +foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) + # Exclude comments + if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") + continue() + endif() + + string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY}) + string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY}) + + # Skip blank or malformed lines + if(${_LIB_VERSION} STREQUAL "") + continue() + endif() + + # For debugging + #message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}") + + set(${_LIB_NAME} "${_LIB_VERSION}") +endforeach() + +if(DEFINED ENV{MEGASEARCH_BOOST_URL}) + set(BOOST_SOURCE_URL "$ENV{MEGASEARCH_BOOST_URL}") +else() + string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) + set(BOOST_SOURCE_URL + "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" + ) +endif() + +if(DEFINED ENV{MEGASEARCH_BZIP2_URL}) + set(BZIP2_SOURCE_URL "$ENV{MEGASEARCH_BZIP2_URL}") +else() + set(BZIP2_SOURCE_URL "https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_EASYLOGGINGPP_URL}) + set(EASYLOGGINGPP_SOURCE_URL "$ENV{MEGASEARCH_EASYLOGGINGPP_URL}") +else() + set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_FAISS_URL}) + set(FAISS_SOURCE_URL "$ENV{MEGASEARCH_FAISS_URL}") +else() + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_GTEST_URL}) + set(GTEST_SOURCE_URL "$ENV{MEGASEARCH_GTEST_URL}") +else () + set(GTEST_SOURCE_URL + "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_LAPACK_URL}) + set(LAPACK_SOURCE_URL "$ENV{MEGASEARCH_LAPACK_URL}") +else() + set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_LZ4_URL}) + set(LZ4_SOURCE_URL "$ENV{MEGASEARCH_LZ4_URL}") +else() + set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_OPENBLAS_URL}) + set(OPENBLAS_SOURCE_URL "$ENV{MEGASEARCH_OPENBLAS_URL}") +else () + set(OPENBLAS_SOURCE_URL + "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_PROMETHEUS_URL}) + set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") +else () + set(PROMETHEUS_SOURCE_URL + "https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_ROCKSDB_URL}) + set(ROCKSDB_SOURCE_URL "$ENV{MEGASEARCH_ROCKSDB_URL}") +else () + set(ROCKSDB_SOURCE_URL + "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SNAPPY_URL}) + set(SNAPPY_SOURCE_URL "$ENV{MEGASEARCH_SNAPPY_URL}") +else() + set(SNAPPY_SOURCE_URL + "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SQLITE_URL}) + set(SQLITE_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_URL}") +else() + set(SQLITE_SOURCE_URL + "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SQLITE_ORM_URL}) + set(SQLITE_ORM_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_ORM_URL}") +else() + set(SQLITE_ORM_SOURCE_URL + "https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip") +endif() + +if(DEFINED ENV{MEGASEARCH_THRIFT_URL}) + set(THRIFT_SOURCE_URL "$ENV{MEGASEARCH_THRIFT_URL}") +else() + set(THRIFT_SOURCE_URL + "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_YAMLCPP_URL}) + set(YAMLCPP_SOURCE_URL "$ENV{MEGASEARCH_YAMLCPP_URL}") +else() + set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_ZLIB_URL}) + set(ZLIB_SOURCE_URL "$ENV{MEGASEARCH_ZLIB_URL}") +else() + set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_ZSTD_URL}) + set(ZSTD_SOURCE_URL "$ENV{MEGASEARCH_ZSTD_URL}") +else() + set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz") +endif() + +# ---------------------------------------------------------------------- +# Add Boost dependencies (code adapted from Apache Kudu (incubating)) + +set(Boost_USE_MULTITHREADED ON) +if(MSVC AND MEGASEARCH_USE_STATIC_CRT) + set(Boost_USE_STATIC_RUNTIME ON) +endif() +set(Boost_ADDITIONAL_VERSIONS + "1.70.0" + "1.70" + "1.69.0" + "1.69" + "1.68.0" + "1.68" + "1.67.0" + "1.67" + "1.66.0" + "1.66" + "1.65.0" + "1.65" + "1.64.0" + "1.64" + "1.63.0" + "1.63" + "1.62.0" + "1.61" + "1.61.0" + "1.62" + "1.60.0" + "1.60") + +if(MEGASEARCH_BOOST_VENDORED) + set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") + set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") + set(BOOST_BUILD_LINK "static") + set(BOOST_STATIC_SYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_STATIC_FILESYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_STATIC_SERIALIZATION_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_SYSTEM_LIBRARY boost_system_static) + set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) + set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) + + if(MEGASEARCH_BOOST_HEADER_ONLY) + set(BOOST_BUILD_PRODUCTS) + set(BOOST_CONFIGURE_COMMAND "") + set(BOOST_BUILD_COMMAND "") else() - find_package(GTest REQUIRED) - set(GTEST_VENDORED 0) + set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} + ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}) + set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" + "--with-libraries=filesystem,serialization,system") + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(BOOST_BUILD_VARIANT "debug") + else() + set(BOOST_BUILD_VARIANT "release") + endif() + set(BOOST_BUILD_COMMAND + "./b2" + "link=${BOOST_BUILD_LINK}" + "variant=${BOOST_BUILD_VARIANT}" + "cxxflags=-fPIC") + + add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") + + add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") + + add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") + + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}) endif() + externalproject_add(boost_ep + URL + ${BOOST_SOURCE_URL} + BUILD_BYPRODUCTS + ${BOOST_BUILD_PRODUCTS} + BUILD_IN_SOURCE + 1 + CONFIGURE_COMMAND + ${BOOST_CONFIGURE_COMMAND} + BUILD_COMMAND + ${BOOST_BUILD_COMMAND} + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") + set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") + add_dependencies(boost_system_static boost_ep) + add_dependencies(boost_filesystem_static boost_ep) + add_dependencies(boost_serialization_static boost_ep) - message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") - message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +else() + if(MSVC) + # disable autolinking in boost + add_definitions(-DBOOST_ALL_NO_LIB) + endif() + +# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) +# # In older versions of CMake (such as 3.2), the system paths for Boost will +# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT +# set(Boost_NO_SYSTEM_PATHS ON) +# endif() + + if(MEGASEARCH_BOOST_USE_SHARED) + # Find shared Boost libraries. + set(Boost_USE_STATIC_LIBS OFF) + set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS ON) + + if(MSVC) + # force all boost libraries to dynamic link + add_definitions(-DBOOST_ALL_DYN_LINK) + endif() + + if(MEGASEARCH_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS serialization system filesystem REQUIRED) + set(BOOST_SYSTEM_LIBRARY Boost::system) + set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) + set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) + endif() + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) + unset(BUILD_SHARED_LIBS_KEEP) + else() + # Find static boost headers and libs + # TODO Differentiate here between release and debug builds + set(Boost_USE_STATIC_LIBS ON) + if(MEGASEARCH_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS serialization system filesystem REQUIRED) + set(BOOST_SYSTEM_LIBRARY Boost::system) + set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) + set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) + endif() + endif() +endif() + +#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) +#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) + +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) +link_directories(SYSTEM ${BOOST_LIB_DIR}) + +# ---------------------------------------------------------------------- +# bzip2 + +macro(build_bzip2) + message(STATUS "Building BZip2-${BZIP2_VERSION} from source") + set(BZIP2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix/src/bzip2_ep") + set(BZIP2_INCLUDE_DIR "${BZIP2_PREFIX}/include") + set(BZIP2_STATIC_LIB + "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") + + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + add_library(bzip2 STATIC IMPORTED) + set_target_properties( + bzip2 + PROPERTIES IMPORTED_LOCATION "${BZIP2_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") + + add_dependencies(bzip2 bzip2_ep) +endmacro() + +if(MEGASEARCH_WITH_BZ2) + resolve_dependency(BZip2) + + if(NOT TARGET bzip2) + add_library(bzip2 UNKNOWN IMPORTED) + set_target_properties(bzip2 + PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") + endif() + link_directories(SYSTEM ${BZIP2_PREFIX}/lib/) + include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# Easylogging++ + +macro(build_easyloggingpp) + message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source") + set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep") + set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include") + set(EASYLOGGINGPP_STATIC_LIB + "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(EASYLOGGINGPP_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -Dtest=OFF + -Dbuild_static_lib=ON) + + externalproject_add(easyloggingpp_ep + URL + ${EASYLOGGINGPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${EASYLOGGINGPP_CMAKE_ARGS} + BUILD_BYPRODUCTS + ${EASYLOGGINGPP_STATIC_LIB}) + + file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") + add_library(easyloggingpp STATIC IMPORTED) + set_target_properties( + easyloggingpp + PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}") + + add_dependencies(easyloggingpp easyloggingpp_ep) +endmacro() + +if(MEGASEARCH_WITH_EASYLOGGINGPP) + resolve_dependency(Easylogging++) + + get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib") + include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# OpenBLAS + +macro(build_openblas) + message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") + set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") + set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") + set(OPENBLAS_STATIC_LIB + "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") + + externalproject_add(openblas_ep + URL + ${OPENBLAS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + INSTALL_COMMAND + ${MAKE} + PREFIX=${OPENBLAS_PREFIX} + install + BUILD_BYPRODUCTS + ${OPENBLAS_STATIC_LIB}) + + file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") + add_library(openblas STATIC IMPORTED) + set_target_properties( + openblas + PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}") + + add_dependencies(openblas openblas_ep) +endmacro() + +#if(MEGASEARCH_WITH_OPENBLAS) +# resolve_dependency(OpenBLAS) +# +# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) +# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") +#endif() + +# ---------------------------------------------------------------------- +# LAPACK + +macro(build_lapack) + message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") + set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") + set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") + set(LAPACK_STATIC_LIB + "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(LAPACK_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib) + + externalproject_add(lapack_ep + URL + ${LAPACK_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${LAPACK_CMAKE_ARGS} + BUILD_BYPRODUCTS + ${LAPACK_STATIC_LIB}) + + file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") + add_library(lapack STATIC IMPORTED) + set_target_properties( + lapack + PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}") + + add_dependencies(lapack lapack_ep) +endmacro() + +#if(MEGASEARCH_WITH_LAPACK) +# resolve_dependency(LAPACK) +# +# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) +# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") +#endif() + +# ---------------------------------------------------------------------- +# FAISS + +macro(build_faiss) + message(STATUS "Building FAISS-${FAISS_VERSION} from source") + set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") + set(FAISS_STATIC_LIB + "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") + +# add_custom_target(faiss_dependencies) +# add_dependencies(faiss_dependencies openblas_ep) +# add_dependencies(faiss_dependencies openblas) +# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) +# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) + + set(FAISS_CONFIGURE_ARGS + "--prefix=${FAISS_PREFIX}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}" + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" + --without-python) + +# if(OPENBLAS_STATIC_LIB) +# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) +# else() +# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) +# endif() +# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) + + if(${MEGASEARCH_WITH_FAISS_GPU_VERSION} STREQUAL "ON") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" + "--with-cuda-arch=${MEGASEARCH_FAISS_GPU_ARCH}") + else() + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) + endif() + + externalproject_add(faiss_ep + URL + ${FAISS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} +# BINARY_DIR +# ${FAISS_PREFIX} +# INSTALL_DIR +# ${FAISS_PREFIX} +# BUILD_COMMAND +# ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 +# INSTALL_DIR +# ${FAISS_PREFIX} +# INSTALL_COMMAND +# "" + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) +# DEPENDS +# ${faiss_dependencies}) + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep) + ExternalProject_Add_StepDependencies(faiss_ep build lapack_ep) + + file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") + add_library(faiss STATIC IMPORTED) + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}") + + add_dependencies(faiss faiss_ep) +endmacro() + +if(MEGASEARCH_WITH_FAISS) + + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + + resolve_dependency(FAISS) + get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") + link_directories(SYSTEM ${FAISS_PREFIX}/lib/) +endif() + +# ---------------------------------------------------------------------- +# Google gtest + +macro(build_gtest) + message(STATUS "Building gtest-${GTEST_VERSION} from source") + set(GTEST_VENDORED TRUE) + set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") + + if(APPLE) + set(GTEST_CMAKE_CXX_FLAGS + ${GTEST_CMAKE_CXX_FLAGS} + -DGTEST_USE_OWN_TR1_TUPLE=1 + -Wno-unused-value + -Wno-ignored-attributes) + endif() + + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_MAIN_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(GTEST_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" + "-DCMAKE_INSTALL_LIBDIR=lib" + -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} + -DCMAKE_BUILD_TYPE=Release) + + set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GMOCK_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + + ExternalProject_Add(googletest_ep + URL + ${GTEST_SOURCE_URL} + BUILD_BYPRODUCTS + ${GTEST_STATIC_LIB} + ${GTEST_MAIN_STATIC_LIB} + ${GMOCK_STATIC_LIB} + CMAKE_ARGS + ${GTEST_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) + + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") add_library(gtest STATIC IMPORTED) - set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB}) + set_target_properties(gtest + PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gtest_main STATIC IMPORTED) - set_target_properties(gtest_main PROPERTIES IMPORTED_LOCATION - ${GTEST_MAIN_STATIC_LIB}) + set_target_properties(gtest_main + PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + + add_library(gmock STATIC IMPORTED) + set_target_properties(gmock + PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") - if(GTEST_VENDORED) - add_dependencies(gtest googletest) - add_dependencies(gtest_main googletest) + add_dependencies(gtest googletest_ep) + add_dependencies(gtest_main googletest_ep) + add_dependencies(gmock googletest_ep) + +endmacro() + +if (MEGASEARCH_BUILD_TESTS) + #message(STATUS "Resolving gtest dependency") + resolve_dependency(GTest) + + if(NOT GTEST_VENDORED) + endif() + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM "${GTEST_PREFIX}/lib") + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# lz4 + +macro(build_lz4) + message(STATUS "Building lz4-${LZ4_VERSION} from source") + set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") + set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/") + + if(MSVC) + if(MEGASEARCH_USE_STATIC_CRT) + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") + else() + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") + endif() + endif() + set(LZ4_STATIC_LIB + "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") + set(LZ4_BUILD_COMMAND + BUILD_COMMAND + msbuild.exe + /m + /p:Configuration=${CMAKE_BUILD_TYPE} + /p:Platform=x64 + /p:PlatformToolset=v140 + ${LZ4_RUNTIME_LIBRARY_LINKAGE} + /t:Build + ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) + else() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + #set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh + # "AR=${CMAKE_AR}") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) + endif() + + # We need to copy the header in lib to directory outside of the build + externalproject_add(lz4_ep + URL + ${LZ4_SOURCE_URL} + ${EP_LOG_OPTIONS} + UPDATE_COMMAND + ${CMAKE_COMMAND} + -E + copy_directory + "${LZ4_BUILD_DIR}/lib" + "${LZ4_PREFIX}/include" + ${LZ4_PATCH_COMMAND} + CONFIGURE_COMMAND + "" + INSTALL_COMMAND + "" + BINARY_DIR + ${LZ4_BUILD_DIR} + BUILD_BYPRODUCTS + ${LZ4_STATIC_LIB} + ${LZ4_BUILD_COMMAND}) + + file(MAKE_DIRECTORY "${LZ4_PREFIX}/include") + add_library(lz4 STATIC IMPORTED) + set_target_properties(lz4 + PROPERTIES IMPORTED_LOCATION "${LZ4_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include") + add_dependencies(lz4 lz4_ep) +endmacro() + +if(MEGASEARCH_WITH_LZ4) + resolve_dependency(Lz4) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/) + include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# Prometheus + +macro(build_prometheus) + message(STATUS "Building Prometheus-${PROMETHEUS_VERSION} from source") + set(PROMETHEUS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix/src/prometheus_ep") + set(PROMETHEUS_STATIC_LIB_NAME prometheus-cpp) + set(PROMETHEUS_CORE_STATIC_LIB + "${PROMETHEUS_PREFIX}/core/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-core${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROMETHEUS_PUSH_STATIC_LIB + "${PROMETHEUS_PREFIX}/push/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-push${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROMETHEUS_PULL_STATIC_LIB + "${PROMETHEUS_PREFIX}/pull/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-pull${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(PROMETHEUS_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF + "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}") + + externalproject_add(prometheus_ep + URL + ${PROMETHEUS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${PROMETHEUS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} + "DESTDIR=${PROMETHEUS_PREFIX}" + install + BUILD_BYPRODUCTS + "${PROMETHEUS_CORE_STATIC_LIB}" + "${PROMETHEUS_PUSH_STATIC_LIB}" + "${PROMETHEUS_PULL_STATIC_LIB}") + + #file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/include") + + add_library(prometheus-cpp-push STATIC IMPORTED) + set_target_properties(prometheus-cpp-push + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PUSH_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/push/include") + add_dependencies(prometheus-cpp-push prometheus_ep) + + add_library(prometheus-cpp-pull STATIC IMPORTED) + set_target_properties(prometheus-cpp-pull + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PULL_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/pull/include") + add_dependencies(prometheus-cpp-pull prometheus_ep) + + add_library(prometheus-cpp-core STATIC IMPORTED) + set_target_properties(prometheus-cpp-core + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_CORE_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/core/include") + add_dependencies(prometheus-cpp-core prometheus_ep) +endmacro() + +if(MEGASEARCH_WITH_PROMETHEUS) + + resolve_dependency(Prometheus) + + # TODO: Don't use global includes but rather target_include_directories + #get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES) + + #get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) + + #get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) + + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) +endif() + +# ---------------------------------------------------------------------- +# RocksDB + +macro(build_rocksdb) + message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source") + set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep") + set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include") + set(ROCKSDB_STATIC_LIB_NAME rocksdb) + set(ROCKSDB_STATIC_LIB + "${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + externalproject_add(rocksdb_ep + URL + ${ROCKSDB_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + static_lib + "prefix=${ROCKSDB_PREFIX}" + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} + install-static + "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" + BUILD_BYPRODUCTS + "${ROCKSDB_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include") + + add_library(rocksdb STATIC IMPORTED) + set_target_properties(rocksdb + PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}") + add_dependencies(rocksdb rocksdb_ep) +endmacro() + +if(MEGASEARCH_WITH_ROCKSDB) + + resolve_dependency(RocksDB) + + # TODO: Don't use global includes but rather target_include_directories +# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) + include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) +endif() + +# ---------------------------------------------------------------------- +# Snappy + +macro(build_snappy) + message(STATUS "Building snappy-${SNAPPY_VERSION} from source") + set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep-prefix/src/snappy_ep") + set(SNAPPY_STATIC_LIB_NAME snappy) + set(SNAPPY_STATIC_LIB + "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(SNAPPY_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_LIBDIR=lib + -DSNAPPY_BUILD_TESTS=OFF + "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") + + externalproject_add(snappy_ep + ${EP_LOG_OPTIONS} + BUILD_IN_SOURCE + 1 + INSTALL_DIR + ${SNAPPY_PREFIX} + URL + ${SNAPPY_SOURCE_URL} + CMAKE_ARGS + ${SNAPPY_CMAKE_ARGS} + BUILD_BYPRODUCTS + "${SNAPPY_STATIC_LIB}") + + file(MAKE_DIRECTORY "${SNAPPY_PREFIX}/include") + + add_library(snappy STATIC IMPORTED) + set_target_properties(snappy + PROPERTIES IMPORTED_LOCATION "${SNAPPY_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${SNAPPY_PREFIX}/include") + add_dependencies(snappy snappy_ep) +endmacro() + +if(MEGASEARCH_WITH_SNAPPY) +# if(Snappy_SOURCE STREQUAL "AUTO") +# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy +# # errornously places them in ${CMAKE_ROOT}/Modules/ +# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong +# # location. +# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 +# # https://src.fedoraproject.org/rpms/snappy/pull-request/1 +# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") +# if(NOT Snappy_FOUND) +# find_package(SnappyAlt) +# endif() +# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) +# build_snappy() +# endif() +# elseif(Snappy_SOURCE STREQUAL "BUNDLED") +# build_snappy() +# elseif(Snappy_SOURCE STREQUAL "SYSTEM") +# # SnappyConfig.cmake is not installed on Ubuntu/Debian +# # TODO: Make a bug report upstream +# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") +# if(NOT Snappy_FOUND) +# find_package(SnappyAlt REQUIRED) +# endif() +# endif() + + resolve_dependency(Snappy) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/) + include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) +endif() + +# ---------------------------------------------------------------------- +# SQLite + +macro(build_sqlite) + message(STATUS "Building SQLITE-${SQLITE_VERSION} from source") + set(SQLITE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix/src/sqlite_ep") + set(SQLITE_INCLUDE_DIR "${SQLITE_PREFIX}/include") + set(SQLITE_STATIC_LIB + "${SQLITE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite3${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(SQLITE_CONFIGURE_ARGS + "--prefix=${SQLITE_PREFIX}" + "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}") + + externalproject_add(sqlite_ep + URL + ${SQLITE_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${SQLITE_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + BUILD_BYPRODUCTS + "${SQLITE_STATIC_LIB}") + + file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}") + add_library(sqlite STATIC IMPORTED) + set_target_properties( + sqlite + PROPERTIES IMPORTED_LOCATION "${SQLITE_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_INCLUDE_DIR}") + + add_dependencies(sqlite sqlite_ep) +endmacro() + +if(MEGASEARCH_WITH_SQLITE) + resolve_dependency(SQLite) + include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}") + link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) +endif() + +# ---------------------------------------------------------------------- +# SQLite_ORM + +macro(build_sqlite_orm) + message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source") + + set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix") + set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz") #sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz + if (NOT EXISTS ${SQLITE_ORM_TAR_NAME}) + file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX}) + file(DOWNLOAD https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz + ${SQLITE_ORM_TAR_NAME}) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME} + WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) + + endif () + set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm") + + #set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep") + #set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm") + +# set(SQLITE_ORM_STATIC_LIB +# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}") +# +# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14") +# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14") +# +# set(SQLITE_ORM_CMAKE_ARGS +# ${EP_COMMON_CMAKE_ARGS} +# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}" +# #"LDFLAGS=-L${SQLITE_PREFIX}" +# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include" +# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include" +# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}" +# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}" +# -DSqliteOrm_BuildTests=off +# -DBUILD_TESTING=off) +# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}") +# +# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}") + +# externalproject_add(sqlite_orm_ep +# URL +# ${SQLITE_ORM_SOURCE_URL} +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix +# CONFIGURE_COMMAND +# "" +# BUILD_COMMAND +# "" +# INSTALL_COMMAND +# "" + #${EP_LOG_OPTIONS} + #${EP_LOG_OPTIONS} +# CMAKE_ARGS +# ${SQLITE_ORM_CMAKE_ARGS} +# BUILD_COMMAND +# ${MAKE} +# ${MAKE_BUILD_ARGS} +# #"LDFLAGS=-L${SQLITE_PREFIX}" +# BUILD_IN_SOURCE +# 1 +# BUILD_BYPRODUCTS +# "${SQLITE_ORM_STATIC_LIB}" +# ) +# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep) + + #set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h) +# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}") +# add_library(sqlite_orm STATIC IMPORTED) +## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}") +# set_target_properties( +# sqlite_orm +# PROPERTIES +# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}") +# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR}) +# target_link_libraries(sqlite_orm INTERFACE sqlite) +# +# add_dependencies(sqlite_orm sqlite_orm_ep) +endmacro() + +if(MEGASEARCH_WITH_SQLITE_ORM) + resolve_dependency(SQLite_ORM) +# ExternalProject_Get_Property(sqlite_orm_ep source_dir) +# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep) + include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") + #message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# Thrift + +macro(build_thrift) + message(STATUS "Building Apache Thrift-${THRIFT_VERSION} from source") + set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep") + set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") + set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") + set(THRIFT_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" + "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DBUILD_EXAMPLES=OFF + -DBUILD_TUTORIALS=OFF + -DWITH_QT4=OFF + -DWITH_C_GLIB=OFF + -DWITH_JAVA=OFF + -DWITH_PYTHON=OFF + -DWITH_HASKELL=OFF + -DWITH_CPP=ON + -DWITH_STATIC_LIB=ON + -DWITH_LIBEVENT=OFF + -DCMAKE_BUILD_TYPE=Release) + + # Thrift also uses boost. Forward important boost settings if there were ones passed. + if(DEFINED BOOST_ROOT) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}") + endif() + if(DEFINED Boost_NAMESPACE) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}") + endif() + + set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") + if(MSVC) + if(MEGASEARCH_USE_STATIC_CRT) + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") + else() + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") + endif() + endif() + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + endif() + set(THRIFT_STATIC_LIB + "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + if(ZLIB_SHARED_LIB) + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" ${THRIFT_CMAKE_ARGS}) + else() + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" ${THRIFT_CMAKE_ARGS}) + endif() + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) + + if(MSVC) + set(WINFLEXBISON_VERSION 2.4.9) + set(WINFLEXBISON_PREFIX + "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") + externalproject_add( + winflexbison_ep + URL + https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip + URL_HASH + MD5=a2e979ea9928fbf8567e995e9c0df765 + SOURCE_DIR + ${WINFLEXBISON_PREFIX} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + "" + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) + + set(THRIFT_CMAKE_ARGS + "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" + "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" + "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" + "-DWITH_SHARED_LIB=OFF" + "-DWITH_PLUGIN=OFF" + ${THRIFT_CMAKE_ARGS}) + elseif(APPLE) + # Some other process always resets BISON_EXECUTABLE to the system default, + # thus we use our own variable here. + if(NOT DEFINED THRIFT_BISON_EXECUTABLE) + find_package(BISON 2.5.1) + + # In the case where we cannot find a system-wide installation, look for + # homebrew and ask for its bison installation. + if(NOT BISON_FOUND) + find_program(BREW_BIN brew) + if(BREW_BIN) + execute_process(COMMAND ${BREW_BIN} --prefix bison + OUTPUT_VARIABLE BISON_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") + find_package(BISON 2.5.1) + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + else() + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + endif() + set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" + ${THRIFT_CMAKE_ARGS}) + endif() + + externalproject_add(thrift_ep + URL + ${THRIFT_SOURCE_URL} + BUILD_BYPRODUCTS + "${THRIFT_STATIC_LIB}" + "${THRIFT_COMPILER}" + CMAKE_ARGS + ${THRIFT_CMAKE_ARGS} + DEPENDS + ${THRIFT_DEPENDENCIES} + ${EP_LOG_OPTIONS}) + + add_library(thrift STATIC IMPORTED) + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${THRIFT_INCLUDE_DIR}") + set_target_properties(thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + add_dependencies(thrift thrift_ep) +endmacro() + +if(MEGASEARCH_WITH_THRIFT) + resolve_dependency(Thrift) + # TODO: Don't use global includes but rather target_include_directories +# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/) + link_directories(SYSTEM ${THRIFT_PREFIX}/lib/) + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# yaml-cpp + +macro(build_yamlcpp) + message(STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source") + set(YAMLCPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix/src/yaml-cpp_ep") + set(YAMLCPP_STATIC_LIB "${YAMLCPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(YAMLCPP_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${YAMLCPP_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -DYAML_CPP_BUILD_TESTS=OFF + -DYAML_CPP_BUILD_TOOLS=OFF) + + externalproject_add(yaml-cpp_ep + URL + ${YAMLCPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_BYPRODUCTS + "${YAMLCPP_STATIC_LIB}" + CMAKE_ARGS + ${YAMLCPP_CMAKE_ARGS}) + + file(MAKE_DIRECTORY "${YAMLCPP_PREFIX}/include") + + add_library(yaml-cpp STATIC IMPORTED) + set_target_properties(yaml-cpp + PROPERTIES IMPORTED_LOCATION "${YAMLCPP_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${YAMLCPP_PREFIX}/include") + + add_dependencies(yaml-cpp yaml-cpp_ep) +endmacro() + +if(MEGASEARCH_WITH_YAMLCPP) + resolve_dependency(yaml-cpp) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) + include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# zlib + +macro(build_zlib) + message(STATUS "Building ZLIB-${ZLIB_VERSION} from source") + set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep") + if(MSVC) + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) + else() + set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) + endif() + else() + set(ZLIB_STATIC_LIB_NAME libz.a) endif() -endif() \ No newline at end of file + set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") + set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) + + externalproject_add(zlib_ep + URL + ${ZLIB_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_BYPRODUCTS + "${ZLIB_STATIC_LIB}" + CMAKE_ARGS + ${ZLIB_CMAKE_ARGS}) + + file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include") + + add_library(zlib STATIC IMPORTED) + set_target_properties(zlib + PROPERTIES IMPORTED_LOCATION "${ZLIB_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include") + + add_dependencies(zlib zlib_ep) +endmacro() + +if(MEGASEARCH_WITH_ZLIB) + resolve_dependency(ZLIB) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# zstd + +macro(build_zstd) + message(STATUS "Building zstd-${ZSTD_VERSION} from source") + set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep") + + set(ZSTD_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DZSTD_BUILD_PROGRAMS=off + -DZSTD_BUILD_SHARED=off + -DZSTD_BUILD_STATIC=on + -DZSTD_MULTITHREAD_SUPPORT=off) + + if(MSVC) + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib") + if(MEGASEARCH_USE_STATIC_CRT) + set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") + endif() + else() + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") + # Only pass our C flags on Unix as on MSVC it leads to a + # "incompatible command-line options" error + set(ZSTD_CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) + endif() + + if(CMAKE_VERSION VERSION_LESS 3.7) + message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") + endif() + + externalproject_add(zstd_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + SOURCE_SUBDIR + "build/cmake" + INSTALL_DIR + ${ZSTD_PREFIX} + URL + ${ZSTD_SOURCE_URL} + BUILD_BYPRODUCTS + "${ZSTD_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ZSTD_PREFIX}/include") + + add_library(zstd STATIC IMPORTED) + set_target_properties(zstd + PROPERTIES IMPORTED_LOCATION "${ZSTD_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_PREFIX}/include") + + add_dependencies(zstd zstd_ep) +endmacro() + +if(MEGASEARCH_WITH_ZSTD) + resolve_dependency(ZSTD) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${ZSTD_PREFIX}/lib) + include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) +endif() diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 964a23fd48c47e8e7df85edac3052d6949382adf..523b1f99681d49a83d241c66eca1d51b4d075110 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,7 +1,7 @@ server_config: address: 0.0.0.0 port: 33001 - transfer_protocol: json #optional: binary, compact, json, debug + transfer_protocol: binary #optional: binary, compact, json server_mode: thread_pool #optional: simple, thread_pool gpu_index: 0 #which gpu to be used diff --git a/cpp/conf/server_config_template.yaml b/cpp/conf/server_config_template.yaml index fb6f6beae249ddf1cc863877af9aa6772c14f888..b701956498ede14ad9a1c867c27a2df6409949ed 100644 --- a/cpp/conf/server_config_template.yaml +++ b/cpp/conf/server_config_template.yaml @@ -1,7 +1,7 @@ server_config: address: 0.0.0.0 port: 33001 - transfer_protocol: json #optional: binary, compact, json, debug + transfer_protocol: binary #optional: binary, compact, json server_mode: thread_pool #optional: simple, thread_pool gpu_index: 0 #which gpu to be used diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d1369b5c0ce6c712cc13823bf9c91161201422c0..ecce311592f80522071dfb3e2ad68dba2e406574 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -47,11 +47,52 @@ set(vecwise_engine_files set(get_sys_info_files license/GetSysInfo.cpp) - include_directories(/usr/include) include_directories(/usr/local/cuda/include) include_directories(thrift/gen-cpp) +#target_link_libraries(megasearch boost_system_static) +#target_link_libraries(megasearch boost_filesystem_static) +#target_link_libraries(megasearch boost_serialization_static) +#target_link_libraries(megasearch bzip2) +#target_link_libraries(megasearch easyloggingpp) +#target_link_libraries(megasearch faiss) +#target_link_libraries(megasearch gtest) +#target_link_libraries(megasearch lapack) +#target_link_libraries(megasearch lz4) +#target_link_libraries(megasearch openblas) +#target_link_libraries(megasearch rocksdb) +#target_link_libraries(megasearch snappy) +#target_link_libraries(megasearch sqlite) +#target_link_libraries(megasearch sqlite_orm) +#target_link_libraries(megasearch thrift) +#target_link_libraries(megasearch yaml-cpp) +#target_link_libraries(megasearch zlib) +#target_link_libraries(megasearch zstd) + +set(third_party_libs + easyloggingpp + sqlite + # sqlite_orm + thrift + yaml-cpp + faiss + lapack + openblas + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + rocksdb + boost_system_static + boost_filesystem_static + boost_serialization_static + bzip2 + lz4 + snappy + zlib + zstd + ) + if (GPU_VERSION STREQUAL "ON") link_directories(/usr/local/cuda/lib64) set(engine_libs @@ -100,13 +141,13 @@ endif () cuda_add_library(vecwise_engine STATIC ${vecwise_engine_files}) -target_link_libraries(vecwise_engine ${engine_libs}) +target_link_libraries(vecwise_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) if (ENABLE_LICENSE STREQUAL "ON") add_library(vecwise_license STATIC ${license_check_files}) - target_link_libraries(vecwise_license ${license_libs}) + target_link_libraries(vecwise_license ${license_libs} ${third_party_libs}) endif () #set(metrics_lib @@ -118,17 +159,6 @@ endif () #add_library(vecwise_engine STATIC ${metrics_files} ) #target_link_libraries(metrics ${metrics_lib}) -add_executable(vecwise_server - ${config_files} - ${server_files} - ${utils_files} - ${service_files} - ${metrics_files} - ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc - ) - - - set(server_libs vecwise_engine librocksdb.a @@ -146,16 +176,30 @@ set(server_libs metrics ) +add_executable(vecwise_server + ${config_files} + ${server_files} + ${utils_files} + ${service_files} + ${metrics_files} + #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc + ) + if (ENABLE_LICENSE STREQUAL "ON") - target_link_libraries(vecwise_server ${server_libs} vecwise_license) + target_link_libraries(vecwise_server ${server_libs} vecwise_license ${third_party_libs}) else () - target_link_libraries(vecwise_server ${server_libs}) + target_link_libraries(vecwise_server ${server_libs} ${third_party_libs}) endif() if (ENABLE_LICENSE STREQUAL "ON") + add_executable(get_sys_info ${get_sys_info_files}) add_executable(license_generator ${license_generator_files}) - target_link_libraries(get_sys_info ${license_libs} vecwise_license) - target_link_libraries(license_generator ${license_libs}) + + target_link_libraries(get_sys_info ${license_libs} vecwise_license ${third_party_libs}) + target_link_libraries(license_generator ${license_libs} ${third_party_libs}) + + install(TARGETS get_sys_info DESTINATION bin) + install(TARGETS license_generator DESTINATION bin) endif () install(TARGETS vecwise_server DESTINATION bin) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index 600eecb4adc64c81628be5b0850b93a9677a1813..74f34d86153257f5f961334e433fdd9b6813db79 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -28,8 +28,7 @@ add_library(megasearch_sdk STATIC link_directories(../../third_party/build/lib) target_link_libraries(megasearch_sdk - libthrift.a - pthread + ${third_party_libs} ) add_subdirectory(examples) diff --git a/cpp/src/sdk/examples/simple/CMakeLists.txt b/cpp/src/sdk/examples/simple/CMakeLists.txt index 5a249bd60993eec4ab7e03933b1d0406b66b2673..3b51a6ae864a01fd5c89757ff131f09ead92144b 100644 --- a/cpp/src/sdk/examples/simple/CMakeLists.txt +++ b/cpp/src/sdk/examples/simple/CMakeLists.txt @@ -8,14 +8,12 @@ aux_source_directory(src src_files) include_directories(src) include_directories(../../megasearch_sdk/include) -include_directories(/usr/include) link_directories(${CMAKE_BINARY_DIR}/megasearch_sdk) add_executable(sdk_simple ./main.cpp ${src_files} - ${service_files} ) target_link_libraries(sdk_simple diff --git a/cpp/src/sdk/examples/simple/main.cpp b/cpp/src/sdk/examples/simple/main.cpp index 2701f9e6600d9d6e4cf98b0fb38258448dd5f2fd..722d40de4fdccb1e959e33526a09d292020abf40 100644 --- a/cpp/src/sdk/examples/simple/main.cpp +++ b/cpp/src/sdk/examples/simple/main.cpp @@ -19,7 +19,8 @@ main(int argc, char *argv[]) { printf("Client start...\n"); std::string app_name = basename(argv[0]); - static struct option long_options[] = {{"conf_file", optional_argument, 0, 'c'}, + static struct option long_options[] = {{"server", optional_argument, 0, 's'}, + {"port", optional_argument, 0, 'p'}, {"help", no_argument, 0, 'h'}, {NULL, 0, 0, 0}}; @@ -28,9 +29,9 @@ main(int argc, char *argv[]) { app_name = argv[0]; int value; - while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) { + while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) { switch (value) { - case 'h': { + case 's': { char *address_ptr = strdup(optarg); address = address_ptr; free(address_ptr); @@ -38,12 +39,14 @@ main(int argc, char *argv[]) { } case 'p': { char *port_ptr = strdup(optarg); - address = port_ptr; + port = port_ptr; free(port_ptr); break; } + case 'h': default: - break; + print_help(app_name); + return EXIT_SUCCESS; } } @@ -58,7 +61,8 @@ void print_help(const std::string &app_name) { printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); printf(" Options:\n"); - printf(" -h Megasearch server address\n"); - printf(" -p Megasearch server port\n"); + printf(" -s --server Server address, default 127.0.0.1\n"); + printf(" -p --port Server port, default 33001\n"); + printf(" -h --help Print help information\n"); printf("\n"); } \ No newline at end of file diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 01669680fce188e7cb53dee93d077c3863e7e9ae..d094e272f1d890a708926916e376638f9c9ae634 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -13,13 +13,59 @@ using namespace megasearch; namespace { + std::string GetTableName(); + + static const std::string TABLE_NAME = GetTableName(); + static const std::string VECTOR_COLUMN_NAME = "face_vector"; + static const std::string ID_COLUMN_NAME = "aid"; + static const std::string CITY_COLUMN_NAME = "city"; + static constexpr int64_t TABLE_DIMENSION = 512; + static constexpr int64_t TOTAL_ROW_COUNT = 100000; + static constexpr int64_t TOP_K = 10; + static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different + +#define BLOCK_SPLITER std::cout << "===========================================" << std::endl; + void PrintTableSchema(const megasearch::TableSchema& tb_schema) { - std::cout << "===========================================" << std::endl; + BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl; std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl; std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl; - std::cout << "===========================================" << std::endl; + BLOCK_SPLITER + } + + void PrintRecordIdArray(const std::vector& record_ids) { + BLOCK_SPLITER + std::cout << "Returned id array count: " << record_ids.size() << std::endl; +#if 0 + for(auto id : record_ids) { + std::cout << std::to_string(id) << std::endl; + } +#endif + BLOCK_SPLITER + } + + void PrintSearchResult(const std::vector& topk_query_result_array) { + BLOCK_SPLITER + std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; + + int32_t index = 0; + for(auto& result : topk_query_result_array) { + index++; + std::cout << "No." << std::to_string(index) << " vector top " + << std::to_string(result.query_result_arrays.size()) + << " search result:" << std::endl; + for(auto& item : result.query_result_arrays) { + std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score); + for(auto& attribute : item.column_map) { + std::cout << "\t" << attribute.first << ":" << attribute.second; + } + std::cout << std::endl; + } + } + + BLOCK_SPLITER } std::string CurrentTime() { @@ -40,9 +86,24 @@ namespace { return s_id; } - static const std::string TABLE_NAME = GetTableName(); - static const std::string VECTOR_COLUMN_NAME = "face_vector"; - static const int64_t TABLE_DIMENSION = 512; + TableSchema BuildTableSchema() { + TableSchema tb_schema; + VectorColumn col1; + col1.name = VECTOR_COLUMN_NAME; + col1.dimension = TABLE_DIMENSION; + col1.store_raw_vector = true; + tb_schema.vector_column_array.emplace_back(col1); + + Column col2 = {ColumnType::int8, ID_COLUMN_NAME}; + tb_schema.attribute_column_array.emplace_back(col2); + + Column col3 = {ColumnType::int16, CITY_COLUMN_NAME}; + tb_schema.attribute_column_array.emplace_back(col3); + + tb_schema.table_name = TABLE_NAME; + + return tb_schema; + } void BuildVectors(int64_t from, int64_t to, std::vector* vector_record_array, @@ -58,6 +119,19 @@ namespace { query_record_array->clear(); } + static const std::map CITY_MAP = { + {0, "Beijing"}, + {1, "Shanhai"}, + {2, "Hangzhou"}, + {3, "Guangzhou"}, + {4, "Shenzheng"}, + {5, "Wuhan"}, + {6, "Chengdu"}, + {7, "Chongqin"}, + {8, "Tianjing"}, + {9, "Hongkong"}, + }; + for (int64_t k = from; k < to; k++) { std::vector f_p; @@ -69,12 +143,16 @@ namespace { if(vector_record_array) { RowRecord record; record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p)); + record.attribute_map[ID_COLUMN_NAME] = std::to_string(k); + record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size()); vector_record_array->emplace_back(record); } if(query_record_array) { QueryRecord record; record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p)); + record.selected_column_array.push_back(ID_COLUMN_NAME); + record.selected_column_array.push_back(CITY_COLUMN_NAME); query_record_array->emplace_back(record); } } @@ -84,61 +162,80 @@ namespace { void ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = Connection::Create(); - ConnectParam param = { address, port }; - conn->Connect(param); - {//create table - TableSchema tb_schema; - VectorColumn col1; - col1.name = VECTOR_COLUMN_NAME; - col1.dimension = TABLE_DIMENSION; - col1.store_raw_vector = true; - tb_schema.vector_column_array.emplace_back(col1); + {//connect server + ConnectParam param = {address, port}; + Status stat = conn->Connect(param); + std::cout << "Connect function call status: " << stat.ToString() << std::endl; + } - Column col2; - col2.name = "age"; - tb_schema.attribute_column_array.emplace_back(col2); + {//server version + std::string version = conn->ServerVersion(); + std::cout << "MegaSearch server version: " << version << std::endl; + } - tb_schema.table_name = TABLE_NAME; + {//sdk version + std::string version = conn->ClientVersion(); + std::cout << "SDK version: " << version << std::endl; + } + + { + std::vector tables; + Status stat = conn->ShowTables(tables); + std::cout << "ShowTables function call status: " << stat.ToString() << std::endl; + std::cout << "All tables: " << std::endl; + for(auto& table : tables) { + std::cout << "\t" << table << std::endl; + } + } + {//create table + TableSchema tb_schema = BuildTableSchema(); PrintTableSchema(tb_schema); Status stat = conn->CreateTable(tb_schema); - std::cout << "Create table result: " << stat.ToString() << std::endl; + std::cout << "CreateTable function call status: " << stat.ToString() << std::endl; } {//describe table TableSchema tb_schema; Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); - std::cout << "Describe table result: " << stat.ToString() << std::endl; + std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl; PrintTableSchema(tb_schema); } {//add vectors std::vector record_array; - BuildVectors(0, 10000, &record_array, nullptr); + BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr); std::vector record_ids; - std::cout << "Begin add vectors" << std::endl; Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids); - std::cout << "Add vector result: " << stat.ToString() << std::endl; - std::cout << "Returned vector ids: " << record_ids.size() << std::endl; + std::cout << "AddVector function call status: " << stat.ToString() << std::endl; + PrintRecordIdArray(record_ids); } {//search vectors + std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl; sleep(10); std::vector record_array; - BuildVectors(500, 510, nullptr, &record_array); + BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array); std::vector topk_query_result_array; - std::cout << "Begin search vectors" << std::endl; - Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, 10); - std::cout << "Search vector result: " << stat.ToString() << std::endl; - std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; + Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K); + std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; + PrintSearchResult(topk_query_result_array); } // {//delete table // Status stat = conn->DeleteTable(TABLE_NAME); -// std::cout << "Delete table result: " << stat.ToString() << std::endl; +// std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; // } + {//server status + std::string status = conn->ServerStatus(); + std::cout << "Server status before disconnect: " << status << std::endl; + } Connection::Destroy(conn); + {//server status + std::string status = conn->ServerStatus(); + std::cout << "Server status after disconnect: " << status << std::endl; + } } \ No newline at end of file diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 25f9ac8c1dcdc8eff62af9c5053213e273d20b9f..f9b01685a0d95b8f0a7e114bc157fe098fde153e 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -14,7 +14,8 @@ enum class StatusCode { OK = 0, Invalid = 1, UnknownError = 2, - NotSupported = 3 + NotSupported = 3, + NotConnected = 4 }; /** diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 90d55ca1e062d3084a9dcb11591d38a1055ed59f..1208494fb379009c1ca0e30364962582200af9a2 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -16,32 +16,50 @@ ClientProxy::ClientPtr() const { return client_ptr; } +bool ClientProxy::IsConnected() const { + return (client_ptr != nullptr && connected_); +} + Status ClientProxy::Connect(const ConnectParam ¶m) { Disconnect(); int32_t port = atoi(param.port.c_str()); - return ClientPtr()->Connect(param.ip_address, port, "json"); + Status status = ClientPtr()->Connect(param.ip_address, port, THRIFT_PROTOCOL_BINARY); + if(status.ok()) { + connected_ = true; + } + + return status; } Status ClientProxy::Connect(const std::string &uri) { Disconnect(); - return Status::NotSupported("Connect interface is not supported."); + size_t index = uri.find_first_of(":", 0); + if((index == std::string::npos)) { + return Status::Invalid("Invalid uri"); + } + + ConnectParam param; + param.ip_address = uri.substr(0, index); + param.port = uri.substr(index + 1); + + return Connect(param); } Status ClientProxy::Connected() const { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { std::string info; ClientPtr()->interface()->Ping(info, ""); } catch ( std::exception& ex) { - return Status(StatusCode::UnknownError, "connection lost: " + std::string(ex.what())); + return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); } return Status::OK(); @@ -49,22 +67,23 @@ ClientProxy::Connected() const { Status ClientProxy::Disconnect() { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } + connected_ = false; return ClientPtr()->Disconnect(); } std::string ClientProxy::ClientVersion() const { - return std::string("Current Version"); + return std::string("v1.0"); } Status ClientProxy::CreateTable(const TableSchema ¶m) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -103,8 +122,8 @@ ClientProxy::CreateTable(const TableSchema ¶m) { Status ClientProxy::CreateTablePartition(const CreateTablePartitionParam ¶m) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -132,8 +151,8 @@ ClientProxy::CreateTablePartition(const CreateTablePartitionParam ¶m) { Status ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam ¶m) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -152,8 +171,8 @@ ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam ¶m) { Status ClientProxy::DeleteTable(const std::string &table_name) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -170,8 +189,8 @@ Status ClientProxy::AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -205,8 +224,8 @@ ClientProxy::SearchVector(const std::string &table_name, const std::vector &query_record_array, std::vector &topk_query_result_array, int64_t topk) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -253,8 +272,8 @@ ClientProxy::SearchVector(const std::string &table_name, Status ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -290,8 +309,8 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch Status ClientProxy::ShowTables(std::vector &table_array) { - if(client_ptr == nullptr) { - return Status(StatusCode::UnknownError, "not connected"); + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); } try { @@ -306,7 +325,7 @@ ClientProxy::ShowTables(std::vector &table_array) { std::string ClientProxy::ServerVersion() const { - if(client_ptr == nullptr) { + if(!IsConnected()) { return ""; } @@ -321,8 +340,8 @@ ClientProxy::ServerVersion() const { std::string ClientProxy::ServerStatus() const { - if(client_ptr == nullptr) { - return "not connected"; + if(!IsConnected()) { + return "not connected to server"; } try { diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index d10571753433676e304dd04ff4b48cae86c131f4..5a96b915449ff8ae6fb3b10048859764a2911eef 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -51,9 +51,11 @@ public: private: std::shared_ptr& ClientPtr() const; + bool IsConnected() const; + private: mutable std::shared_ptr client_ptr; - + bool connected_ = false; }; } diff --git a/cpp/src/sdk/src/client/ThriftClient.cpp b/cpp/src/sdk/src/client/ThriftClient.cpp index d492ee69d4528b99ccb5823c69253d98480ed863..5ab000c200c5363493fa2f3ee8fce0182b11da76 100644 --- a/cpp/src/sdk/src/client/ThriftClient.cpp +++ b/cpp/src/sdk/src/client/ThriftClient.cpp @@ -50,14 +50,12 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin stdcxx::shared_ptr socket_ptr(new transport::TSocket(address, port)); stdcxx::shared_ptr transport_ptr(new TBufferedTransport(socket_ptr)); stdcxx::shared_ptr protocol_ptr; - if(protocol == "binary") { + if(protocol == THRIFT_PROTOCOL_BINARY) { protocol_ptr.reset(new TBinaryProtocol(transport_ptr)); - } else if(protocol == "json") { + } else if(protocol == THRIFT_PROTOCOL_JSON) { protocol_ptr.reset(new TJSONProtocol(transport_ptr)); - } else if(protocol == "compact") { + } else if(protocol == THRIFT_PROTOCOL_COMPACT) { protocol_ptr.reset(new TCompactProtocol(transport_ptr)); - } else if(protocol == "debug") { - protocol_ptr.reset(new TDebugProtocol(transport_ptr)); } else { //CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently"; return Status(StatusCode::Invalid, "unsupported protocol"); @@ -67,7 +65,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin client_ = std::make_shared(protocol_ptr); } catch ( std::exception& ex) { //CLIENT_LOG_ERROR << "connect encounter exception: " << ex.what(); - return Status(StatusCode::UnknownError, "failed to connect megasearch server" + std::string(ex.what())); + return Status(StatusCode::NotConnected, "failed to connect megasearch server" + std::string(ex.what())); } return Status::OK(); diff --git a/cpp/src/sdk/src/client/ThriftClient.h b/cpp/src/sdk/src/client/ThriftClient.h index ccfca6b47c15266de4c4e47c52f8cde836b8c368..4834835edf44e1ecddb49f2438375d4186ff963a 100644 --- a/cpp/src/sdk/src/client/ThriftClient.h +++ b/cpp/src/sdk/src/client/ThriftClient.h @@ -14,6 +14,10 @@ namespace megasearch { using MegasearchServiceClientPtr = std::shared_ptr; +static const std::string THRIFT_PROTOCOL_JSON = "json"; +static const std::string THRIFT_PROTOCOL_BINARY = "binary"; +static const std::string THRIFT_PROTOCOL_COMPACT = "compact"; + class ThriftClient { public: ThriftClient(); diff --git a/cpp/src/sdk/src/interface/Status.cpp b/cpp/src/sdk/src/interface/Status.cpp index 58a65133c489abc66c158ff106abe7142b3a0789..6ac3b7ec9391038738b6d0b7d8117fce372ea0c5 100644 --- a/cpp/src/sdk/src/interface/Status.cpp +++ b/cpp/src/sdk/src/interface/Status.cpp @@ -101,6 +101,8 @@ std::string Status::CodeAsString() const { break; case StatusCode::NotSupported: type = "Not Supported"; break; + case StatusCode::NotConnected: type = "Not Connected"; + break; default: type = "Unknown"; break; } diff --git a/cpp/src/server/MegasearchHandler.cpp b/cpp/src/server/MegasearchHandler.cpp index 54310d72bedff3be9060602571e40f587793985b..4e7c57560637c7d30b4ece0a59791719b4a3155b 100644 --- a/cpp/src/server/MegasearchHandler.cpp +++ b/cpp/src/server/MegasearchHandler.cpp @@ -32,14 +32,14 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) { void MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam ¶m) { - // Your implementation goes here - printf("CreateTablePartition\n"); + BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param); + MegasearchScheduler::ExecTask(task_ptr); } void MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam ¶m) { - // Your implementation goes here - printf("DeleteTablePartition\n"); + BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param); + MegasearchScheduler::ExecTask(task_ptr); } void @@ -67,14 +67,14 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std: void MegasearchServiceHandler::ShowTables(std::vector &_return) { - // Your implementation goes here - printf("ShowTables\n"); + BaseTaskPtr task_ptr = ShowTablesTask::Create(_return); + MegasearchScheduler::ExecTask(task_ptr); } void MegasearchServiceHandler::Ping(std::string& _return, const std::string& cmd) { - // Your implementation goes here - printf("Ping\n"); + BaseTaskPtr task_ptr = PingTask::Create(cmd, _return); + MegasearchScheduler::ExecTask(task_ptr); } } diff --git a/cpp/src/server/MegasearchServer.cpp b/cpp/src/server/MegasearchServer.cpp index 5940d20bbce1c6a64ef6d2fb8e5e62b59233e7d9..f771fc4dd835ee0ff4f2538a7fb20b03ecc6fb6d 100644 --- a/cpp/src/server/MegasearchServer.cpp +++ b/cpp/src/server/MegasearchServer.cpp @@ -54,7 +54,6 @@ MegasearchServer::StartService() { stdcxx::shared_ptr server_transport(new TServerSocket(address, port)); stdcxx::shared_ptr transport_factory(new TBufferedTransportFactory()); - std::string protocol = "json"; stdcxx::shared_ptr protocol_factory; if (protocol == "binary") { protocol_factory.reset(new TBinaryProtocolFactory()); @@ -62,8 +61,6 @@ MegasearchServer::StartService() { protocol_factory.reset(new TJSONProtocolFactory()); } else if (protocol == "compact") { protocol_factory.reset(new TCompactProtocolFactory()); - } else if (protocol == "debug") { - protocol_factory.reset(new TDebugProtocolFactory()); } else { //SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently"; return; diff --git a/cpp/src/server/MegasearchTask.cpp b/cpp/src/server/MegasearchTask.cpp index 6f01b38ce19ec8a5da9a9f03a44a678ae4a26a41..68718933483a5b76876982bee2742fc8ba1395b9 100644 --- a/cpp/src/server/MegasearchTask.cpp +++ b/cpp/src/server/MegasearchTask.cpp @@ -21,6 +21,7 @@ namespace server { static const std::string DQL_TASK_GROUP = "dql"; static const std::string DDL_DML_TASK_GROUP = "ddl_dml"; +static const std::string PING_TASK_GROUP = "ping"; static const std::string VECTOR_UID = "uid"; static const uint64_t USE_MT = 5000; @@ -48,6 +49,10 @@ namespace { } } + ~DBWrapper() { + delete db_; + } + zilliz::vecwise::engine::DB* DB() { return db_; } private: @@ -78,17 +83,17 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) { ServerError CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); - + try { if(schema_.vector_column_array.empty()) { return SERVER_INVALID_ARGUMENT; } IVecIdMapper::GetInstance()->AddGroup(schema_.table_name); - engine::meta::TableSchema table_schema; - table_schema.dimension = (uint16_t)schema_.vector_column_array[0].dimension; - table_schema.table_id = schema_.table_name; - engine::Status stat = DB()->CreateTable(table_schema); + engine::meta::TableSchema table_info; + table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension; + table_info.table_id = schema_.table_name; + engine::Status stat = DB()->CreateTable(table_info); if(!stat.ok()) {//could exist error_msg_ = "Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; @@ -109,7 +114,7 @@ ServerError CreateTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DescribeTableTask::DescribeTableTask(const std::string &table_name, thrift::TableSchema &schema) - : BaseTask(DDL_DML_TASK_GROUP), + : BaseTask(PING_TASK_GROUP), table_name_(table_name), schema_(schema) { schema_.table_name = table_name_; @@ -123,9 +128,9 @@ ServerError DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { - engine::meta::TableSchema table_schema; - table_schema.table_id = table_name_; - engine::Status stat = DB()->DescribeTable(table_schema); + engine::meta::TableSchema table_info; + table_info.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -154,8 +159,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name) } -BaseTaskPtr DeleteTableTask::Create(const std::string& table_id) { - return std::shared_ptr(new DeleteTableTask(table_id)); +BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) { + return std::shared_ptr(new DeleteTableTask(group_id)); } ServerError DeleteTableTask::OnExecute() { @@ -168,6 +173,60 @@ ServerError DeleteTableTask::OnExecute() { return SERVER_NOT_IMPLEMENT; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam ¶m) + : BaseTask(DDL_DML_TASK_GROUP), + param_(param) { + +} + +BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam ¶m) { + return std::shared_ptr(new CreateTablePartitionTask(param)); +} + +ServerError CreateTablePartitionTask::OnExecute() { + error_code_ = SERVER_NOT_IMPLEMENT; + error_msg_ = "create table partition not implemented"; + SERVER_LOG_ERROR << error_msg_; + + return SERVER_NOT_IMPLEMENT; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam ¶m) + : BaseTask(DDL_DML_TASK_GROUP), + param_(param) { + +} + +BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam ¶m) { + return std::shared_ptr(new DeleteTablePartitionTask(param)); +} + +ServerError DeleteTablePartitionTask::OnExecute() { + error_code_ = SERVER_NOT_IMPLEMENT; + error_msg_ = "delete table partition not implemented"; + SERVER_LOG_ERROR << error_msg_; + + return SERVER_NOT_IMPLEMENT; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +ShowTablesTask::ShowTablesTask(std::vector& tables) + : BaseTask(PING_TASK_GROUP), + tables_(tables) { + +} + +BaseTaskPtr ShowTablesTask::Create(std::vector& tables) { + return std::shared_ptr(new ShowTablesTask(tables)); +} + +ServerError ShowTablesTask::OnExecute() { + IVecIdMapper::GetInstance()->AllGroups(tables_); + + return SERVER_SUCCESS; +} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// AddVectorTask::AddVectorTask(const std::string& table_name, @@ -178,7 +237,6 @@ AddVectorTask::AddVectorTask(const std::string& table_name, record_array_(record_array), record_ids_(record_ids) { record_ids_.clear(); - record_ids_.resize(record_array.size()); } BaseTaskPtr AddVectorTask::Create(const std::string& table_name, @@ -195,9 +253,9 @@ ServerError AddVectorTask::OnExecute() { return SERVER_SUCCESS; } - engine::meta::TableSchema table_schema; - table_schema.table_id = table_name_; - engine::Status stat = DB()->DescribeTable(table_schema); + engine::meta::TableSchema table_info; + table_info.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -208,7 +266,7 @@ ServerError AddVectorTask::OnExecute() { rc.Record("get group info"); uint64_t vec_count = (uint64_t)record_array_.size(); - uint64_t group_dim = table_schema.dimension; + uint64_t group_dim = table_info.dimension; std::vector vec_f; vec_f.resize(vec_count*group_dim);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { @@ -228,6 +286,7 @@ ServerError AddVectorTask::OnExecute() { return error_code_; } + //convert double array to float array(thrift has no float type) const double* d_p = reinterpret_cast(record.vector_map.begin()->second.data()); for(uint64_t d = 0; d < vec_dim; d++) { vec_f[i*vec_dim + d] = (float)(d_p[d]); @@ -245,12 +304,27 @@ ServerError AddVectorTask::OnExecute() { return error_code_; } - if(record_ids_.size() < vec_count) { + if(record_ids_.size() != vec_count) { SERVER_LOG_ERROR << "Vector ID not returned"; return SERVER_UNEXPECTED_ERROR; } - rc.Record("done"); + //persist attributes + for(uint64_t i = 0; i < vec_count; i++) { + const auto &record = record_array_[i]; + + //any attributes? + if(record.attribute_map.empty()) { + continue; + } + + std::string nid = std::to_string(record_ids_[i]); + std::string attrib_str; + AttributeSerializer::Encode(record.attribute_map, attrib_str); + IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_); + } + + rc.Record("persist vector attributes"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; @@ -293,9 +367,9 @@ ServerError SearchVectorTask::OnExecute() { return error_code_; } - engine::meta::TableSchema table_schema; - table_schema.table_id = table_name_; - engine::Status stat = DB()->DescribeTable(table_schema); + engine::meta::TableSchema table_info; + table_info.table_id = table_name_; + engine::Status stat = DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_GROUP_NOT_EXIST; error_msg_ = "Engine failed: " + stat.ToString(); @@ -305,7 +379,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector vec_f; uint64_t record_count = (uint64_t)record_array_.size(); - vec_f.resize(record_count*table_schema.dimension); + vec_f.resize(record_count*table_info.dimension); for(uint64_t i = 0; i < record_array_.size(); i++) { const auto& record = record_array_[i]; @@ -317,14 +391,15 @@ ServerError SearchVectorTask::OnExecute() { } uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value? - if (vec_dim != table_schema.dimension) { + if (vec_dim != table_info.dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << table_schema.dimension; + << " vs. group dimension:" << table_info.dimension; error_code_ = SERVER_INVALID_VECTOR_DIMENSION; error_msg_ = "Engine failed: " + stat.ToString(); return error_code_; } + //convert double array to float array(thrift has no float type) const double* d_p = reinterpret_cast(record.vector_map.begin()->second.data()); for(uint64_t d = 0; d < vec_dim; d++) { vec_f[i*vec_dim + d] = (float)(d_p[d]); @@ -336,25 +411,50 @@ ServerError SearchVectorTask::OnExecute() { std::vector dates; engine::QueryResults results; stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results); + rc.Record("search vectors from engine"); if(!stat.ok()) { SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; - } else { - rc.Record("do searching"); - for(engine::QueryResult& result : results){ - thrift::TopKQueryResult thrift_topk_result; - for(auto id : result) { - thrift::QueryResult thrift_result; - thrift_result.__set_id(id); + } + + if(results.size() != record_count) { + SERVER_LOG_ERROR << "Search result not returned"; + return SERVER_UNEXPECTED_ERROR; + } + + //construct result array + for(uint64_t i = 0; i < record_count; i++) { + auto& result = results[i]; + const auto& record = record_array_[i]; + + thrift::TopKQueryResult thrift_topk_result; + for(auto id : result) { + thrift::QueryResult thrift_result; + thrift_result.__set_id(id); + + //need get attributes? + if(record.selected_column_array.empty()) { thrift_topk_result.query_result_arrays.emplace_back(thrift_result); + continue; } - result_array_.emplace_back(thrift_topk_result); + std::string nid = std::to_string(id); + std::string attrib_str; + IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_); + + AttribMap attrib_map; + AttributeSerializer::Decode(attrib_str, attrib_map); + + for(auto& attribute : record.selected_column_array) { + thrift_result.column_map[attribute] = attrib_map[attribute]; + } + + thrift_topk_result.query_result_arrays.emplace_back(thrift_result); } - rc.Record("construct result"); - } - rc.Record("done"); + result_array_.emplace_back(thrift_topk_result); + } + rc.Record("construct result"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; @@ -366,6 +466,26 @@ ServerError SearchVectorTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +PingTask::PingTask(const std::string& cmd, std::string& result) + : BaseTask(PING_TASK_GROUP), + cmd_(cmd), + result_(result) { + +} + +BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) { + return std::shared_ptr(new PingTask(cmd, result)); +} + +ServerError PingTask::OnExecute() { + if(cmd_ == "version") { + result_ = "v1.2.0";//currently hardcode + } + + return SERVER_SUCCESS; +} + } } } diff --git a/cpp/src/server/MegasearchTask.h b/cpp/src/server/MegasearchTask.h index 26c23e1b262bedeaa33d25814ff6caac130d10be..af4178eb2154530460f1f771c231e2426baf72ad 100644 --- a/cpp/src/server/MegasearchTask.h +++ b/cpp/src/server/MegasearchTask.h @@ -65,6 +65,50 @@ private: std::string table_name_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class CreateTablePartitionTask : public BaseTask { +public: + static BaseTaskPtr Create(const thrift::CreateTablePartitionParam ¶m); + +protected: + CreateTablePartitionTask(const thrift::CreateTablePartitionParam ¶m); + + ServerError OnExecute() override; + + +private: + const thrift::CreateTablePartitionParam ¶m_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class DeleteTablePartitionTask : public BaseTask { +public: + static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam ¶m); + +protected: + DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam ¶m); + + ServerError OnExecute() override; + + +private: + const thrift::DeleteTablePartitionParam ¶m_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class ShowTablesTask : public BaseTask { +public: + static BaseTaskPtr Create(std::vector& tables); + +protected: + ShowTablesTask(std::vector& tables); + + ServerError OnExecute() override; + +private: + std::vector& tables_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class AddVectorTask : public BaseTask { public: @@ -108,6 +152,21 @@ private: std::vector& result_array_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class PingTask : public BaseTask { +public: + static BaseTaskPtr Create(const std::string& cmd, std::string& result); + +protected: + PingTask(const std::string& cmd, std::string& result); + + ServerError OnExecute() override; + +private: + std::string cmd_; + std::string& result_; +}; + } } } \ No newline at end of file diff --git a/cpp/src/server/RocksIdMapper.cpp b/cpp/src/server/RocksIdMapper.cpp index 6eec10c60d7f9123d06eba1ce67d3a8ab9be27b8..3ac887a61087af1c16dd03d74b0d7dd2386eca43 100644 --- a/cpp/src/server/RocksIdMapper.cpp +++ b/cpp/src/server/RocksIdMapper.cpp @@ -9,6 +9,7 @@ #include "utils/Log.h" #include "utils/CommonUtil.h" +#include "rocksdb/db.h" #include "rocksdb/slice.h" #include "rocksdb/options.h" @@ -108,6 +109,19 @@ bool RocksIdMapper::IsGroupExist(const std::string& group) const { return IsGroupExistInternal(group); } +ServerError RocksIdMapper::AllGroups(std::vector& groups) const { + groups.clear(); + + std::lock_guard lck(db_mutex_); + for(auto& pair : column_handles_) { + if(pair.first == ROCKSDB_DEFAULT_GROUP) { + continue; + } + groups.push_back(pair.first); + } + + return SERVER_SUCCESS; +} ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { std::lock_guard lck(db_mutex_); diff --git a/cpp/src/server/RocksIdMapper.h b/cpp/src/server/RocksIdMapper.h index 1ffee7f3353e1978783f9b377b1315b03177e3a6..777fbf08a6645c646c60d90911ddb9c690cdbddc 100644 --- a/cpp/src/server/RocksIdMapper.h +++ b/cpp/src/server/RocksIdMapper.h @@ -8,13 +8,16 @@ #include "utils/Error.h" #include "VecIdMapper.h" -#include "rocksdb/db.h" - #include #include #include #include +namespace rocksdb { + class DB; + class ColumnFamilyHandle; +} + namespace zilliz { namespace vecwise { namespace server { @@ -26,6 +29,7 @@ class RocksIdMapper : public IVecIdMapper{ ServerError AddGroup(const std::string& group) override; bool IsGroupExist(const std::string& group) const override; + ServerError AllGroups(std::vector& groups) const override; ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index a630442c987ad602e169d2951e69500c3b71dfac..8bb387bdd95d81fb9d660d93916517fc8c098878 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -8,7 +8,7 @@ #include "utils/Error.h" #include "config/ConfigNode.h" -#include +#include "yaml-cpp/yaml.h" namespace zilliz { namespace vecwise { diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index d9de2ca3eb5aee5b995e26b414e5c784b768401b..aece541d9105018c6ad558d94f99f0331b69f463 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -15,6 +15,7 @@ #include "rocksdb/options.h" #include +#include namespace zilliz { namespace vecwise { @@ -30,6 +31,30 @@ IVecIdMapper* IVecIdMapper::GetInstance() { #endif } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SimpleIdMapper : public IVecIdMapper{ +public: + SimpleIdMapper(); + ~SimpleIdMapper(); + + ServerError AddGroup(const std::string& group) override; + bool IsGroupExist(const std::string& group) const override; + ServerError AllGroups(std::vector& groups) const override; + + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; + + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; + + ServerError Delete(const std::string& nid, const std::string& group = "") override; + ServerError DeleteGroup(const std::string& group) override; + +private: + using ID_MAPPING = std::unordered_map; + mutable std::unordered_map id_groups_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// SimpleIdMapper::SimpleIdMapper() { @@ -52,6 +77,15 @@ SimpleIdMapper::IsGroupExist(const std::string& group) const { return id_groups_.count(group) > 0; } +ServerError SimpleIdMapper::AllGroups(std::vector& groups) const { + groups.clear(); + + for(auto& pair : id_groups_) { + groups.push_back(pair.first); + } + + return SERVER_SUCCESS; +} //not thread-safe ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index f3c2bdde27d7c3f741a82250cafb9c60defc4949..658eb7f3fb43953b0da4b6b466eb0b057afc4ed9 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -9,11 +9,6 @@ #include #include -#include - -namespace rocksdb { - class DB; -} namespace zilliz { namespace vecwise { @@ -27,6 +22,7 @@ public: virtual ServerError AddGroup(const std::string& group) = 0; virtual bool IsGroupExist(const std::string& group) const = 0; + virtual ServerError AllGroups(std::vector& groups) const = 0; virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; @@ -39,28 +35,6 @@ public: virtual ServerError DeleteGroup(const std::string& group) = 0; }; -class SimpleIdMapper : public IVecIdMapper{ -public: - SimpleIdMapper(); - ~SimpleIdMapper(); - - ServerError AddGroup(const std::string& group) override; - bool IsGroupExist(const std::string& group) const override; - - ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; - ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - - ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; - ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - - ServerError Delete(const std::string& nid, const std::string& group = "") override; - ServerError DeleteGroup(const std::string& group) override; - -private: - using ID_MAPPING = std::unordered_map; - mutable std::unordered_map id_groups_; -}; - } } } diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt new file mode 100644 index 0000000000000000000000000000000000000000..6357f5dfa900e2ff7c522c5495cf808aa4182768 --- /dev/null +++ b/cpp/thirdparty/versions.txt @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Toolchain library versions +# +# This file is used by `download_dependencies.sh` and cmake to figure out which +# version of a dependency to fetch. In order to add a new dependency, add a +# version variable, e.g. MY_DEP_VERSION and append an entry in the +# `DEPENDENCIES` array (see the comment on top of the declaration for the +# format). + +BOOST_VERSION=1.70.0 +BZIP2_VERSION=1.0.6 +EASYLOGGINGPP_VERSION=v9.96.7 +FAISS_VERSION=v1.5.2 +GTEST_VERSION=1.8.1 +LAPACK_VERSION=v3.8.0 +LZ4_VERSION=v1.9.1 +OPENBLAS_VERSION=v0.3.6 +PROMETHEUS_VERSION=master +ROCKSDB_VERSION=v6.0.2 +SNAPPY_VERSION=1.1.7 +SQLITE_VERSION=3280000 +SQLITE_ORM_VERSION=master +THRIFT_VERSION=0.12.0 +YAMLCPP_VERSION=0.6.2 +ZLIB_VERSION=v1.2.11 +ZSTD_VERSION=v1.4.0 + +# The first field is the name of the environment variable expected by cmake. +# This _must_ match what is defined. The second field is the name of the +# generated archive file. The third field is the url of the project for the +# given version. +DEPENDENCIES=( + "MEGASEARCH_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz" + "MEGASEARCH_BZIP2_URL bzip2-${BZIP2_VERSION}.tar.gz https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz" + "MEGASEARCH_EASYLOGGINGPP_URL easyloggingpp-${EASYLOGGINGPP_VERSION}.tar.gz https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz" + "MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz" + "MEGASEARCH_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" + "MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz + "MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz" + "MEGASEARCH_OPENBLAS_URL openblas-${OPENBLAS_VERSION}.tar.gz https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz" + "MEGASEARCH_PROMETHEUS_URL prometheus-${PROMETHEUS_VERSION}.tar.gz https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz" + "MEGASEARCH_ROCKSDB_URL rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" + "MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" + "MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz" + "MEGASEARCH_SQLITE_ORM_URL sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz" + "MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" + "MEGASEARCH_YAMLCPP_URL yaml-cpp-${YAMLCPP_VERSION}.tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" + "MEGASEARCH_ZLIB_URL zlib-${ZLIB_VERSION}.tar.gz https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz" + "MEGASEARCH_ZSTD_URL zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz" + ) + +# vim: set filetype=sh: \ No newline at end of file diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 652fd4927a521d70282d718a6a49b8a60ca69d7a..7d370428962334efe217f5d380be4d6049f73d73 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -5,18 +5,26 @@ #------------------------------------------------------------------------------- link_directories( "${CMAKE_BINARY_DIR}/lib" - "${VECWISE_THIRD_PARTY_BUILD}/lib" + #"${VECWISE_THIRD_PARTY_BUILD}/lib" + "${GTEST_PREFIX}/lib/" ) +message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib") set(unittest_srcs - ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc) + ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp) + #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc) set(unittest_libs yaml-cpp + gtest + gmock gtest_main gmock_main + easyloggingpp pthread - metrics) + metrics + openblas + gfortran) add_subdirectory(server) add_subdirectory(db) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 30fa0f4a20cb5083668364e967dd3679daa357e7..2b3d4609b0ccc4d0f74fe1641cfbc91ac0e68d2e 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -42,4 +42,4 @@ set(db_libs lz4 ) -target_link_libraries(db_test ${unittest_libs} ${db_libs}) +target_link_libraries(db_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index eb179e6b6a5c8c6c6008c0ccbb29e0bf1c7f0b02..1ea5431586b1c2e47d5e344764a8d3d3491ffcbe 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -45,7 +45,7 @@ set(wrapper_libs zstd lz4 ) -target_link_libraries(wrapper_test ${unittest_libs} ${wrapper_libs}) +target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) set(topk_test_src topk_test.cpp diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt index 55509c195c66add2871e1e71acb947850a069b5b..1182d256fcf92581bc7bfdae9bba5d3c8f6cf8f1 100644 --- a/cpp/unittest/license/CMakeLists.txt +++ b/cpp/unittest/license/CMakeLists.txt @@ -16,6 +16,7 @@ link_directories(/usr/local/cuda/lib64) link_directories(/usr/lib/x86_64-linux-gnu) link_directories(/usr/lib/nvidia-415) link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/) +link_directories(/usr/local/cuda/lib64/stubs/) set(require_files ../../src/license/LicenseLibrary.cpp @@ -44,4 +45,4 @@ set(db_libs boost_serialization ) -target_link_libraries(license_test ${unittest_libs} ${db_libs}) +target_link_libraries(license_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 25fed9669499cbecd225c9fd7d6b5acbd5e44115..954f34d141165583c540214975f891ff7847e7b0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -57,7 +57,7 @@ set(count_test_src add_executable(metrics_test ${count_test_src} ${require_files} ) target_link_libraries(metrics_test - ${unittest_libs} + faiss cudart cublas @@ -72,5 +72,5 @@ target_link_libraries(metrics_test # prometheus-cpp-core pthread z - + ${unittest_libs} ) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 5d081d72c9d1de368e27c7ab898650f1282b82ea..1d346604f6ba9ef40b28b19fb820908fb0c90e0b 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -35,23 +35,23 @@ cuda_add_executable(server_test set(require_libs stdc++ - boost_system - boost_filesystem - pthread - snappy - bz2 - z - zstd rocksdb faiss cudart cublas sqlite3 + boost_system + boost_filesystem + snappy + z + bz2 + zstd lz4 + pthread ) target_link_libraries(server_test - ${unittest_libs} ${require_libs} ${cuda_library} + ${unittest_libs} )