diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 37fc0fa1f26a06c5c3361e0758f60e8999c16168..595bd8b4ecf8fbf1d05b760a8259b5a729b34607 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-16 - Implement metrics without prometheus - MS-6 - Implement SDK interface part 1 - MS-21 - Implement SDK interface part 2 +- MS-26 - cmake. Add thirdparty packages ### Task diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 37a397028074c0fa86a26392f1c903ea11d36a19..b6b8ad1aa024e3d1b865c93836606edd65f4ee67 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -4,10 +4,33 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.14) +message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(MEGASEARCH_VERSION "0.1.0") + +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}") + +project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}") project(vecwise_engine LANGUAGES CUDA CXX) +set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}") +set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}") +set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}") + +if(MEGASEARCH_VERSION_MAJOR STREQUAL "" + OR MEGASEARCH_VERSION_MINOR STREQUAL "" + OR MEGASEARCH_VERSION_PATCH STREQUAL "") + message(FATAL_ERROR "Failed to determine MegaSearch version from '${MEGASEARCH_VERSION}'") +endif() + +message(STATUS "MegaSearch version: " + "${MEGASEARCH_VERSION_MAJOR}.${MEGASEARCH_VERSION_MINOR}.${MEGASEARCH_VERSION_PATCH} " + "(full: '${MEGASEARCH_VERSION}')") + +set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR}) + find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") @@ -19,6 +42,7 @@ if (GPU_VERSION STREQUAL "ON") endif () set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") message("building vecwise_engine on x86 architecture") @@ -32,43 +56,48 @@ else() set(VECWISE_BUILD_ARCH unknown) endif() -if(DEFINED UNIX) - message("building vecwise on Unix") - set(VECWISE_BUILD_SYSTEM macos) -elseif(DEFINED APPLE) - message("building vecwise on MacOS") - set(VECWISE_BUILD_SYSTEM unix) -else() - message("unknown OS") - set(VECWISE_BUILD_SYSTEM unknown) -endif () - if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") if (GPU_VERSION STREQUAL "ON") set(ENABLE_LICENSE "ON") add_definitions("-DENABLE_LICENSE") endif () else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH}) + + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +if (BUILD_UNIT_TEST) + option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON) +endif(BUILD_UNIT_TEST) + +include(ExternalProject) +include(DefineOptions) +include(BuildUtils) +include(ThirdPartyPackages) + +include_directories(${MEGASEARCH_SOURCE_DIR}) +link_directories(${MEGASEARCH_BINARY_DIR}) + +## Following should be check set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) -set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) -set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) +#set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) +#set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) add_compile_definitions(PROFILER=${PROFILER}) include_directories(${VECWISE_ENGINE_INCLUDE}) include_directories(${VECWISE_ENGINE_SRC}) -include_directories(${VECWISE_THIRD_PARTY_BUILD}/include) +#include_directories(${VECWISE_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib) -link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64) +#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib) +#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64) #execute_process(COMMAND bash build.sh # WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) @@ -91,3 +120,5 @@ install(FILES conf/vecwise_engine_log.conf DESTINATION conf) + +config_summary() diff --git a/cpp/build.sh b/cpp/build.sh index fde83360ba6443ef5b166f98b46231fa43867906..abab7bdffafcad4188cded1429aad01aeaa786b4 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -72,7 +72,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then make clean fi -make -j || exit 1 +make -j 1 || exit 1 if [[ ${BUILD_TYPE} != "Debug" ]]; then strip src/vecwise_server diff --git a/cpp/cmake/BuildUtils.cmake b/cpp/cmake/BuildUtils.cmake new file mode 100644 index 0000000000000000000000000000000000000000..361d038c2277320f46551b5759358782da0b50e4 --- /dev/null +++ b/cpp/cmake/BuildUtils.cmake @@ -0,0 +1,95 @@ +function(ADD_THIRDPARTY_LIB LIB_NAME) + set(options) + set(one_value_args SHARED_LIB STATIC_LIB) + set(multi_value_args DEPS INCLUDE_DIRECTORIES) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + if(ARG_STATIC_LIB AND ARG_SHARED_LIB) + if(NOT ARG_STATIC_LIB) + message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") + endif() + + set(AUG_LIB_NAME "${LIB_NAME}_static") + add_library(${AUG_LIB_NAME} STATIC IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + + set(AUG_LIB_NAME "${LIB_NAME}_shared") + add_library(${AUG_LIB_NAME} SHARED IMPORTED) + + if(WIN32) + # Mark the ".lib" location as part of a Windows DLL + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") + else() + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") + endif() + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + elseif(ARG_STATIC_LIB) + set(AUG_LIB_NAME "${LIB_NAME}_static") + add_library(${AUG_LIB_NAME} STATIC IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + elseif(ARG_SHARED_LIB) + set(AUG_LIB_NAME "${LIB_NAME}_shared") + add_library(${AUG_LIB_NAME} SHARED IMPORTED) + + if(WIN32) + # Mark the ".lib" location as part of a Windows DLL + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") + else() + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") + endif() + message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") + if(ARG_DEPS) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") + endif() + if(ARG_INCLUDE_DIRECTORIES) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${ARG_INCLUDE_DIRECTORIES}") + endif() + else() + message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") + endif() +endfunction() \ No newline at end of file diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake new file mode 100644 index 0000000000000000000000000000000000000000..3171402cf12303cb2c114d14fda3115c484eaffc --- /dev/null +++ b/cpp/cmake/DefineOptions.cmake @@ -0,0 +1,190 @@ + +macro(set_option_category name) + set(MEGASEARCH_OPTION_CATEGORY ${name}) + list(APPEND "MEGASEARCH_OPTION_CATEGORIES" ${name}) +endmacro() + +macro(define_option name description default) + option(${name} ${description} ${default}) + list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name}) + set("${name}_OPTION_DESCRIPTION" ${description}) + set("${name}_OPTION_DEFAULT" ${default}) + set("${name}_OPTION_TYPE" "bool") +endmacro() + +function(list_join lst glue out) + if("${${lst}}" STREQUAL "") + set(${out} "" PARENT_SCOPE) + return() + endif() + + list(GET ${lst} 0 joined) + list(REMOVE_AT ${lst} 0) + foreach(item ${${lst}}) + set(joined "${joined}${glue}${item}") + endforeach() + set(${out} ${joined} PARENT_SCOPE) +endfunction() + +macro(define_option_string name description default) + set(${name} ${default} CACHE STRING ${description}) + list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name}) + set("${name}_OPTION_DESCRIPTION" ${description}) + set("${name}_OPTION_DEFAULT" "\"${default}\"") + set("${name}_OPTION_TYPE" "string") + + set("${name}_OPTION_ENUM" ${ARGN}) + list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM") + if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + set_property(CACHE ${name} PROPERTY STRINGS ${ARGN}) + endif() +endmacro() + +#---------------------------------------------------------------------- +set_option_category("Thirdparty") + +set(MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT "AUTO") + +define_option_string(MEGASEARCH_DEPENDENCY_SOURCE + "Method to use for acquiring MEGASEARCH's build dependencies" + "${MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM") + +define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD + "Show output from ExternalProjects rather than just logging to files" OFF) + +define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) + +define_option(MEGASEARCH_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ +Note that this requires linking Boost statically" ON) + +define_option(MEGASEARCH_BOOST_HEADER_ONLY "Use only BOOST headers" OFF) + +define_option(MEGASEARCH_WITH_BZ2 "Build with BZ2 compression" ON) + +define_option(MEGASEARCH_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) + +define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON) + +define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON) + +define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against" + "-gencode=arch=compute_61,code=sm_61") + +define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON) + +define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON) + +define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON) + +define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON) + +define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON) + +define_option(MEGASEARCH_WITH_SQLITE "Build with SQLite library" ON) + +define_option(MEGASEARCH_WITH_SQLITE_ORM "Build with SQLite ORM library" ON) + +define_option(MEGASEARCH_WITH_THRIFT "Build with Apache Thrift library" ON) + +define_option(MEGASEARCH_WITH_YAMLCPP "Build with yaml-cpp library" ON) + +define_option(MEGASEARCH_WITH_ZLIB "Build with zlib compression" ON) + +if(CMAKE_VERSION VERSION_LESS 3.7) + set(MEGASEARCH_WITH_ZSTD_DEFAULT OFF) +else() + # ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7. + set(MEGASEARCH_WITH_ZSTD_DEFAULT ON) +endif() +define_option(MEGASEARCH_WITH_ZSTD "Build with zstd compression" ${MEGASEARCH_WITH_ZSTD_DEFAULT}) + +#---------------------------------------------------------------------- +if(MSVC) + set_option_category("MSVC") + + define_option(MSVC_LINK_VERBOSE + "Pass verbose linking options when linking libraries and executables" + OFF) + + define_option(MEGASEARCH_USE_STATIC_CRT "Build MEGASEARCH with statically linked CRT" OFF) +endif() + + +#---------------------------------------------------------------------- +#set_option_category("Test and benchmark") +# +#define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON) + +#---------------------------------------------------------------------- +macro(config_summary) + message(STATUS "---------------------------------------------------------------------") + message(STATUS "MEGASEARCH version: ${MEGASEARCH_VERSION}") + message(STATUS) + message(STATUS "Build configuration summary:") + + message(STATUS " Generator: ${CMAKE_GENERATOR}") + message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") + message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") + if(${CMAKE_EXPORT_COMPILE_COMMANDS}) + message( + STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") + endif() + + foreach(category ${MEGASEARCH_OPTION_CATEGORIES}) + + message(STATUS) + message(STATUS "${category} options:") + + set(option_names ${MEGASEARCH_${category}_OPTION_NAMES}) + + set(max_value_length 0) + foreach(name ${option_names}) + string(LENGTH "\"${${name}}\"" value_length) + if(${max_value_length} LESS ${value_length}) + set(max_value_length ${value_length}) + endif() + endforeach() + + foreach(name ${option_names}) + if("${${name}_OPTION_TYPE}" STREQUAL "string") + set(value "\"${${name}}\"") + else() + set(value "${${name}}") + endif() + + set(default ${${name}_OPTION_DEFAULT}) + set(description ${${name}_OPTION_DESCRIPTION}) + string(LENGTH ${description} description_length) + if(${description_length} LESS 70) + string( + SUBSTRING + " " + ${description_length} -1 description_padding) + else() + set(description_padding " + ") + endif() + + set(comment "[${name}]") + + if("${value}" STREQUAL "${default}") + set(comment "[default] ${comment}") + endif() + + if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + set(comment "${comment} [${${name}_OPTION_ENUM}]") + endif() + + string( + SUBSTRING "${value} " + 0 ${max_value_length} value) + + message(STATUS " ${description} ${description_padding} ${value} ${comment}") + endforeach() + + endforeach() + +endmacro() diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index e85025a427dddbcc232fb2222f71cba211bbedda..121799d9eb9f1e2b3898a55e4bf3780153c8a7cd 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -15,52 +15,1430 @@ # specific language governing permissions and limitations # under the License. -set(GTEST_VERSION "1.8.0") +set(MEGASEARCH_THIRDPARTY_DEPENDENCIES -message(MEGASEARCH_BUILD_TESTS ${MEGASEARCH_BUILD_TESTS}) + BOOST + BZip2 + Easylogging++ + FAISS + GTest + LAPACK + Lz4 + OpenBLAS + RocksDB + Snappy + SQLite + SQLite_ORM + Thrift + yaml-cpp + ZLIB + ZSTD) -if(MEGASEARCH_BUILD_TESTS) - add_custom_target(unittest ctest -L unittest) +message(STATUS "Using ${MEGASEARCH_DEPENDENCY_SOURCE} approach to find dependencies") - if("$ENV{GTEST_HOME}" STREQUAL "") - message("Yes") - set(GTEST_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +# For each dependency, set dependency source to global default, if unset +foreach(DEPENDENCY ${MEGASEARCH_THIRDPARTY_DEPENDENCIES}) + if("${${DEPENDENCY}_SOURCE}" STREQUAL "") + set(${DEPENDENCY}_SOURCE ${MEGASEARCH_DEPENDENCY_SOURCE}) + endif() +endforeach() + +macro(build_dependency DEPENDENCY_NAME) + if("${DEPENDENCY_NAME}" STREQUAL "BZip2") + build_bzip2() + elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++") + build_easyloggingpp() + elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") + build_faiss() + elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") + build_lapack() + elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") + build_lz4() + elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") + build_gtest() + elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") + build_openblas() + elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB") + build_rocksdb() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") + build_snappy() + elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") + build_sqlite() + elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM") + build_sqlite_orm() + elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift") + build_thrift() + elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") + build_yamlcpp() + elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") + build_zlib() + elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") + build_zstd() + else() + message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") + endif () +endmacro() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest") - set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GTEST_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_MAIN_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") +macro(resolve_dependency DEPENDENCY_NAME) + if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + #message(STATUS "Finding ${DEPENDENCY_NAME} package") +# find_package(${DEPENDENCY_NAME} QUIET) +# if (NOT ${DEPENDENCY_NAME}_FOUND) + #message(STATUS "${DEPENDENCY_NAME} package not found") + build_dependency(${DEPENDENCY_NAME}) +# endif () + elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") + build_dependency(${DEPENDENCY_NAME}) + elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + find_package(${DEPENDENCY_NAME} REQUIRED) + endif () +endmacro() - set(GTEST_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX} - -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) +# ---------------------------------------------------------------------- +# ExternalProject options - ExternalProject_Add(googletest - URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" - BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" "${GTEST_MAIN_STATIC_LIB}" - CMAKE_ARGS ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - set(GTEST_VENDORED 1) +string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) + +set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") +set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") + +if(NOT MSVC) + # Set -fPIC on all external projects + set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") + set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") +endif() + +# CC/CXX environment variables are captured on the first invocation of the +# builder (e.g make or ninja) instead of when CMake is invoked into to build +# directory. This leads to issues if the variables are exported in a subshell +# and the invocation of make/ninja is in distinct subshell without the same +# environment (CC/CXX). +set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) + +if(CMAKE_AR) + set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) +endif() + +if(CMAKE_RANLIB) + set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB}) +endif() + +# External projects are still able to override the following declarations. +# cmake command line will favor the last defined variable when a duplicate is +# encountered. This requires that `EP_COMMON_CMAKE_ARGS` is always the first +# argument. +set(EP_COMMON_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) + +if(NOT MEGASEARCH_VERBOSE_THIRDPARTY_BUILD) + set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) +else() + set(EP_LOG_OPTIONS) +endif() + +# Ensure that a default make is set +if("${MAKE}" STREQUAL "") + if(NOT MSVC) + find_program(MAKE make) + endif() +endif() + +# Using make -j in sub-make is fragile +# see discussion https://github.com/apache/MEGASEARCH/pull/2779 +if(${CMAKE_GENERATOR} MATCHES "Makefiles") + set(MAKE_BUILD_ARGS "") +else() + # limit the maximum number of jobs for ninja + set(MAKE_BUILD_ARGS "-j4") +endif() + +# ---------------------------------------------------------------------- +# Find pthreads + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +# ---------------------------------------------------------------------- +# Versions and URLs for toolchain builds, which also can be used to configure +# offline builds + +# Read toolchain versions from cpp/thirdparty/versions.txt +set(THIRDPARTY_DIR "${MEGASEARCH_SOURCE_DIR}/thirdparty") +file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) +foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) + # Exclude comments + if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") + continue() + endif() + + string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY}) + string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY}) + + # Skip blank or malformed lines + if(${_LIB_VERSION} STREQUAL "") + continue() + endif() + + # For debugging + #message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}") + + set(${_LIB_NAME} "${_LIB_VERSION}") +endforeach() + +if(DEFINED ENV{MEGASEARCH_BOOST_URL}) + set(BOOST_SOURCE_URL "$ENV{MEGASEARCH_BOOST_URL}") +else() + string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) + set(BOOST_SOURCE_URL + "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" + ) +endif() + +if(DEFINED ENV{MEGASEARCH_BZIP2_URL}) + set(BZIP2_SOURCE_URL "$ENV{MEGASEARCH_BZIP2_URL}") +else() + set(BZIP2_SOURCE_URL "https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_EASYLOGGINGPP_URL}) + set(EASYLOGGINGPP_SOURCE_URL "$ENV{MEGASEARCH_EASYLOGGINGPP_URL}") +else() + set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_FAISS_URL}) + set(FAISS_SOURCE_URL "$ENV{MEGASEARCH_FAISS_URL}") +else() + set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_GTEST_URL}) + set(GTEST_SOURCE_URL "$ENV{MEGASEARCH_GTEST_URL}") +else () + set(GTEST_SOURCE_URL + "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_LAPACK_URL}) + set(LAPACK_SOURCE_URL "$ENV{MEGASEARCH_LAPACK_URL}") +else() + set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_LZ4_URL}) + set(LZ4_SOURCE_URL "$ENV{MEGASEARCH_LZ4_URL}") +else() + set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_OPENBLAS_URL}) + set(OPENBLAS_SOURCE_URL "$ENV{MEGASEARCH_OPENBLAS_URL}") +else () + set(OPENBLAS_SOURCE_URL + "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") +endif() + +if (DEFINED ENV{MEGASEARCH_ROCKSDB_URL}) + set(ROCKSDB_SOURCE_URL "$ENV{MEGASEARCH_ROCKSDB_URL}") +else () + set(ROCKSDB_SOURCE_URL + "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SNAPPY_URL}) + set(SNAPPY_SOURCE_URL "$ENV{MEGASEARCH_SNAPPY_URL}") +else() + set(SNAPPY_SOURCE_URL + "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SQLITE_URL}) + set(SQLITE_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_URL}") +else() + set(SQLITE_SOURCE_URL + "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_SQLITE_ORM_URL}) + set(SQLITE_ORM_SOURCE_URL "$ENV{MEGASEARCH_SQLITE_ORM_URL}") +else() + set(SQLITE_ORM_SOURCE_URL + "https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip") +endif() + +if(DEFINED ENV{MEGASEARCH_THRIFT_URL}) + set(THRIFT_SOURCE_URL "$ENV{MEGASEARCH_THRIFT_URL}") +else() + set(THRIFT_SOURCE_URL + "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_YAMLCPP_URL}) + set(YAMLCPP_SOURCE_URL "$ENV{MEGASEARCH_YAMLCPP_URL}") +else() + set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_ZLIB_URL}) + set(ZLIB_SOURCE_URL "$ENV{MEGASEARCH_ZLIB_URL}") +else() + set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MEGASEARCH_ZSTD_URL}) + set(ZSTD_SOURCE_URL "$ENV{MEGASEARCH_ZSTD_URL}") +else() + set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz") +endif() + +# ---------------------------------------------------------------------- +# Add Boost dependencies (code adapted from Apache Kudu (incubating)) + +set(Boost_USE_MULTITHREADED ON) +if(MSVC AND MEGASEARCH_USE_STATIC_CRT) + set(Boost_USE_STATIC_RUNTIME ON) +endif() +set(Boost_ADDITIONAL_VERSIONS + "1.70.0" + "1.70" + "1.69.0" + "1.69" + "1.68.0" + "1.68" + "1.67.0" + "1.67" + "1.66.0" + "1.66" + "1.65.0" + "1.65" + "1.64.0" + "1.64" + "1.63.0" + "1.63" + "1.62.0" + "1.61" + "1.61.0" + "1.62" + "1.60.0" + "1.60") + +if(MEGASEARCH_BOOST_VENDORED) + set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") + set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") + set(BOOST_BUILD_LINK "static") + set(BOOST_STATIC_SYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_STATIC_FILESYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_STATIC_SERIALIZATION_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BOOST_SYSTEM_LIBRARY boost_system_static) + set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) + set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) + + if(MEGASEARCH_BOOST_HEADER_ONLY) + set(BOOST_BUILD_PRODUCTS) + set(BOOST_CONFIGURE_COMMAND "") + set(BOOST_BUILD_COMMAND "") else() - find_package(GTest REQUIRED) - set(GTEST_VENDORED 0) + set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} + ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}) + set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" + "--with-libraries=filesystem,serialization,system") + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(BOOST_BUILD_VARIANT "debug") + else() + set(BOOST_BUILD_VARIANT "release") + endif() + set(BOOST_BUILD_COMMAND + "./b2" + "link=${BOOST_BUILD_LINK}" + "variant=${BOOST_BUILD_VARIANT}" + "cxxflags=-fPIC") + + add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") + + add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") + + add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") + + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}) endif() + externalproject_add(boost_ep + URL + ${BOOST_SOURCE_URL} + BUILD_BYPRODUCTS + ${BOOST_BUILD_PRODUCTS} + BUILD_IN_SOURCE + 1 + CONFIGURE_COMMAND + ${BOOST_CONFIGURE_COMMAND} + BUILD_COMMAND + ${BOOST_BUILD_COMMAND} + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") + set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") + add_dependencies(boost_system_static boost_ep) + add_dependencies(boost_filesystem_static boost_ep) + add_dependencies(boost_serialization_static boost_ep) - message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") - message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +else() + if(MSVC) + # disable autolinking in boost + add_definitions(-DBOOST_ALL_NO_LIB) + endif() + +# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) +# # In older versions of CMake (such as 3.2), the system paths for Boost will +# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT +# set(Boost_NO_SYSTEM_PATHS ON) +# endif() + + if(MEGASEARCH_BOOST_USE_SHARED) + # Find shared Boost libraries. + set(Boost_USE_STATIC_LIBS OFF) + set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) + set(BUILD_SHARED_LIBS ON) + + if(MSVC) + # force all boost libraries to dynamic link + add_definitions(-DBOOST_ALL_DYN_LINK) + endif() + + if(MEGASEARCH_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS serialization system filesystem REQUIRED) + set(BOOST_SYSTEM_LIBRARY Boost::system) + set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) + set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) + endif() + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) + unset(BUILD_SHARED_LIBS_KEEP) + else() + # Find static boost headers and libs + # TODO Differentiate here between release and debug builds + set(Boost_USE_STATIC_LIBS ON) + if(MEGASEARCH_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS serialization system filesystem REQUIRED) + set(BOOST_SYSTEM_LIBRARY Boost::system) + set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) + set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) + set(MEGASEARCH_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) + endif() + endif() +endif() + +#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) +#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) + +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) +link_directories(SYSTEM ${BOOST_LIB_DIR}) + +# ---------------------------------------------------------------------- +# bzip2 + +macro(build_bzip2) + message(STATUS "Building BZip2-${BZIP2_VERSION} from source") + set(BZIP2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix/src/bzip2_ep") + set(BZIP2_INCLUDE_DIR "${BZIP2_PREFIX}/include") + set(BZIP2_STATIC_LIB + "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") + + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + add_library(bzip2 STATIC IMPORTED) + set_target_properties( + bzip2 + PROPERTIES IMPORTED_LOCATION "${BZIP2_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") + + add_dependencies(bzip2 bzip2_ep) +endmacro() + +if(MEGASEARCH_WITH_BZ2) + resolve_dependency(BZip2) + + if(NOT TARGET bzip2) + add_library(bzip2 UNKNOWN IMPORTED) + set_target_properties(bzip2 + PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") + endif() + link_directories(SYSTEM ${BZIP2_PREFIX}/lib/) + include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# Easylogging++ + +macro(build_easyloggingpp) + message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source") + set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep") + set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include") + set(EASYLOGGINGPP_STATIC_LIB + "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(EASYLOGGINGPP_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -Dtest=OFF + -Dbuild_static_lib=ON) + + externalproject_add(easyloggingpp_ep + URL + ${EASYLOGGINGPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${EASYLOGGINGPP_CMAKE_ARGS} + BUILD_BYPRODUCTS + ${EASYLOGGINGPP_STATIC_LIB}) + + file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") + add_library(easyloggingpp STATIC IMPORTED) + set_target_properties( + easyloggingpp + PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}") + + add_dependencies(easyloggingpp easyloggingpp_ep) +endmacro() + +if(MEGASEARCH_WITH_EASYLOGGINGPP) + resolve_dependency(Easylogging++) + + get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib") + include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# OpenBLAS + +macro(build_openblas) + message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") + set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") + set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") + set(OPENBLAS_STATIC_LIB + "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") + + externalproject_add(openblas_ep + URL + ${OPENBLAS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + INSTALL_COMMAND + ${MAKE} + PREFIX=${OPENBLAS_PREFIX} + install + BUILD_BYPRODUCTS + ${OPENBLAS_STATIC_LIB}) + + file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") + add_library(openblas STATIC IMPORTED) + set_target_properties( + openblas + PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}") + + add_dependencies(openblas openblas_ep) +endmacro() + +#if(MEGASEARCH_WITH_OPENBLAS) +# resolve_dependency(OpenBLAS) +# +# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) +# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") +#endif() + +# ---------------------------------------------------------------------- +# LAPACK + +macro(build_lapack) + message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") + set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") + set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") + set(LAPACK_STATIC_LIB + "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(LAPACK_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib) + + externalproject_add(lapack_ep + URL + ${LAPACK_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${LAPACK_CMAKE_ARGS} + BUILD_BYPRODUCTS + ${LAPACK_STATIC_LIB}) + + file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") + add_library(lapack STATIC IMPORTED) + set_target_properties( + lapack + PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}") + + add_dependencies(lapack lapack_ep) +endmacro() + +#if(MEGASEARCH_WITH_LAPACK) +# resolve_dependency(LAPACK) +# +# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) +# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") +#endif() + +# ---------------------------------------------------------------------- +# FAISS + +macro(build_faiss) + message(STATUS "Building FAISS-${FAISS_VERSION} from source") + set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") + set(FAISS_STATIC_LIB + "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") + +# add_custom_target(faiss_dependencies) +# add_dependencies(faiss_dependencies openblas_ep) +# add_dependencies(faiss_dependencies openblas) +# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) +# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) + + set(FAISS_CONFIGURE_ARGS + "--prefix=${FAISS_PREFIX}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}" + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" + --without-python) + +# if(OPENBLAS_STATIC_LIB) +# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) +# else() +# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) +# endif() +# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) + + if(${MEGASEARCH_WITH_FAISS_GPU_VERSION} STREQUAL "ON") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" + "--with-cuda-arch=${MEGASEARCH_FAISS_GPU_ARCH}") + else() + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) + endif() + + externalproject_add(faiss_ep + URL + ${FAISS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} +# BINARY_DIR +# ${FAISS_PREFIX} +# INSTALL_DIR +# ${FAISS_PREFIX} +# BUILD_COMMAND +# ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 +# INSTALL_DIR +# ${FAISS_PREFIX} +# INSTALL_COMMAND +# "" + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) +# DEPENDS +# ${faiss_dependencies}) + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep) + ExternalProject_Add_StepDependencies(faiss_ep build lapack_ep) + + file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") + add_library(faiss STATIC IMPORTED) + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}") + + add_dependencies(faiss faiss_ep) +endmacro() + +if(MEGASEARCH_WITH_FAISS) + + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + + resolve_dependency(FAISS) + get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") + link_directories(SYSTEM ${FAISS_PREFIX}/lib/) +endif() + +# ---------------------------------------------------------------------- +# Google gtest + +macro(build_gtest) + message(STATUS "Building gtest-${GTEST_VERSION} from source") + set(GTEST_VENDORED TRUE) + set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") + + if(APPLE) + set(GTEST_CMAKE_CXX_FLAGS + ${GTEST_CMAKE_CXX_FLAGS} + -DGTEST_USE_OWN_TR1_TUPLE=1 + -Wno-unused-value + -Wno-ignored-attributes) + endif() + + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_MAIN_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(GTEST_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" + "-DCMAKE_INSTALL_LIBDIR=lib" + -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} + -DCMAKE_BUILD_TYPE=Release) + + set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GMOCK_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + + ExternalProject_Add(googletest_ep + URL + ${GTEST_SOURCE_URL} + BUILD_BYPRODUCTS + ${GTEST_STATIC_LIB} + ${GTEST_MAIN_STATIC_LIB} + ${GMOCK_STATIC_LIB} + CMAKE_ARGS + ${GTEST_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) + + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") add_library(gtest STATIC IMPORTED) - set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB}) + set_target_properties(gtest + PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gtest_main STATIC IMPORTED) - set_target_properties(gtest_main PROPERTIES IMPORTED_LOCATION - ${GTEST_MAIN_STATIC_LIB}) + set_target_properties(gtest_main + PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + + add_library(gmock STATIC IMPORTED) + set_target_properties(gmock + PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + + add_dependencies(gtest googletest_ep) + add_dependencies(gtest_main googletest_ep) + add_dependencies(gmock googletest_ep) + +endmacro() + +message(STATUS "MEGASEARCH_BUILD_TESTS: ${MEGASEARCH_BUILD_TESTS}") +if (MEGASEARCH_BUILD_TESTS) + #message(STATUS "Resolving gtest dependency") + resolve_dependency(GTest) + + if(NOT GTEST_VENDORED) + endif() + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM "${GTEST_PREFIX}/lib") + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# lz4 + +macro(build_lz4) + message(STATUS "Building lz4-${LZ4_VERSION} from source") + set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") + set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/") + + if(MSVC) + if(MEGASEARCH_USE_STATIC_CRT) + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") + else() + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") + endif() + endif() + set(LZ4_STATIC_LIB + "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") + set(LZ4_BUILD_COMMAND + BUILD_COMMAND + msbuild.exe + /m + /p:Configuration=${CMAKE_BUILD_TYPE} + /p:Platform=x64 + /p:PlatformToolset=v140 + ${LZ4_RUNTIME_LIBRARY_LINKAGE} + /t:Build + ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) + else() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + #set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh + # "AR=${CMAKE_AR}") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) + endif() + + # We need to copy the header in lib to directory outside of the build + externalproject_add(lz4_ep + URL + ${LZ4_SOURCE_URL} + ${EP_LOG_OPTIONS} + UPDATE_COMMAND + ${CMAKE_COMMAND} + -E + copy_directory + "${LZ4_BUILD_DIR}/lib" + "${LZ4_PREFIX}/include" + ${LZ4_PATCH_COMMAND} + CONFIGURE_COMMAND + "" + INSTALL_COMMAND + "" + BINARY_DIR + ${LZ4_BUILD_DIR} + BUILD_BYPRODUCTS + ${LZ4_STATIC_LIB} + ${LZ4_BUILD_COMMAND}) + + file(MAKE_DIRECTORY "${LZ4_PREFIX}/include") + add_library(lz4 STATIC IMPORTED) + set_target_properties(lz4 + PROPERTIES IMPORTED_LOCATION "${LZ4_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include") + add_dependencies(lz4 lz4_ep) +endmacro() + +if(MEGASEARCH_WITH_LZ4) + resolve_dependency(Lz4) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/) + include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# RocksDB + +macro(build_rocksdb) + message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source") + set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep") + set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include") + set(ROCKSDB_STATIC_LIB_NAME rocksdb) + set(ROCKSDB_STATIC_LIB + "${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + externalproject_add(rocksdb_ep + URL + ${ROCKSDB_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + static_lib + "prefix=${ROCKSDB_PREFIX}" + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} + install-static + "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" + BUILD_BYPRODUCTS + "${ROCKSDB_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include") + + add_library(rocksdb STATIC IMPORTED) + set_target_properties(rocksdb + PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}") + add_dependencies(rocksdb rocksdb_ep) +endmacro() + +if(MEGASEARCH_WITH_ROCKSDB) + resolve_dependency(RocksDB) + + # TODO: Don't use global includes but rather target_include_directories +# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) + include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) +endif() + +# ---------------------------------------------------------------------- +# Snappy + +macro(build_snappy) + message(STATUS "Building snappy-${SNAPPY_VERSION} from source") + set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep-prefix/src/snappy_ep") + set(SNAPPY_STATIC_LIB_NAME snappy) + set(SNAPPY_STATIC_LIB + "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(SNAPPY_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_LIBDIR=lib + -DSNAPPY_BUILD_TESTS=OFF + "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") + + externalproject_add(snappy_ep + ${EP_LOG_OPTIONS} + BUILD_IN_SOURCE + 1 + INSTALL_DIR + ${SNAPPY_PREFIX} + URL + ${SNAPPY_SOURCE_URL} + CMAKE_ARGS + ${SNAPPY_CMAKE_ARGS} + BUILD_BYPRODUCTS + "${SNAPPY_STATIC_LIB}") + + file(MAKE_DIRECTORY "${SNAPPY_PREFIX}/include") + + add_library(snappy STATIC IMPORTED) + set_target_properties(snappy + PROPERTIES IMPORTED_LOCATION "${SNAPPY_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${SNAPPY_PREFIX}/include") + add_dependencies(snappy snappy_ep) +endmacro() + +if(MEGASEARCH_WITH_SNAPPY) +# if(Snappy_SOURCE STREQUAL "AUTO") +# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy +# # errornously places them in ${CMAKE_ROOT}/Modules/ +# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong +# # location. +# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 +# # https://src.fedoraproject.org/rpms/snappy/pull-request/1 +# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") +# if(NOT Snappy_FOUND) +# find_package(SnappyAlt) +# endif() +# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) +# build_snappy() +# endif() +# elseif(Snappy_SOURCE STREQUAL "BUNDLED") +# build_snappy() +# elseif(Snappy_SOURCE STREQUAL "SYSTEM") +# # SnappyConfig.cmake is not installed on Ubuntu/Debian +# # TODO: Make a bug report upstream +# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") +# if(NOT Snappy_FOUND) +# find_package(SnappyAlt REQUIRED) +# endif() +# endif() + + resolve_dependency(Snappy) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/) + include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) +endif() + +# ---------------------------------------------------------------------- +# SQLite + +macro(build_sqlite) + message(STATUS "Building SQLITE-${SQLITE_VERSION} from source") + set(SQLITE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix/src/sqlite_ep") + set(SQLITE_INCLUDE_DIR "${SQLITE_PREFIX}/include") + set(SQLITE_STATIC_LIB + "${SQLITE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite3${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(SQLITE_CONFIGURE_ARGS + "--prefix=${SQLITE_PREFIX}" + "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}") + + externalproject_add(sqlite_ep + URL + ${SQLITE_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${SQLITE_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + BUILD_BYPRODUCTS + "${SQLITE_STATIC_LIB}") + + file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}") + add_library(sqlite STATIC IMPORTED) + set_target_properties( + sqlite + PROPERTIES IMPORTED_LOCATION "${SQLITE_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_INCLUDE_DIR}") + + add_dependencies(sqlite sqlite_ep) +endmacro() + +if(MEGASEARCH_WITH_SQLITE) + resolve_dependency(SQLite) + include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}") + link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) +endif() + +# ---------------------------------------------------------------------- +# SQLite_ORM + +macro(build_sqlite_orm) + message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source") + + set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix") + set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz") #sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz + if (NOT EXISTS ${SQLITE_ORM_TAR_NAME}) + file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX}) + file(DOWNLOAD https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz + ${SQLITE_ORM_TAR_NAME}) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME} + WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) + + endif () + set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm") + + #set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep") + #set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm") + +# set(SQLITE_ORM_STATIC_LIB +# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}") +# +# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14") +# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14") +# +# set(SQLITE_ORM_CMAKE_ARGS +# ${EP_COMMON_CMAKE_ARGS} +# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}" +# #"LDFLAGS=-L${SQLITE_PREFIX}" +# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include" +# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include" +# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}" +# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}" +# -DSqliteOrm_BuildTests=off +# -DBUILD_TESTING=off) +# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}") +# +# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}") + +# externalproject_add(sqlite_orm_ep +# URL +# ${SQLITE_ORM_SOURCE_URL} +# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix +# CONFIGURE_COMMAND +# "" +# BUILD_COMMAND +# "" +# INSTALL_COMMAND +# "" + #${EP_LOG_OPTIONS} + #${EP_LOG_OPTIONS} +# CMAKE_ARGS +# ${SQLITE_ORM_CMAKE_ARGS} +# BUILD_COMMAND +# ${MAKE} +# ${MAKE_BUILD_ARGS} +# #"LDFLAGS=-L${SQLITE_PREFIX}" +# BUILD_IN_SOURCE +# 1 +# BUILD_BYPRODUCTS +# "${SQLITE_ORM_STATIC_LIB}" +# ) +# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep) + + #set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h) +# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}") +# add_library(sqlite_orm STATIC IMPORTED) +## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}") +# set_target_properties( +# sqlite_orm +# PROPERTIES +# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}") +# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR}) +# target_link_libraries(sqlite_orm INTERFACE sqlite) +# +# add_dependencies(sqlite_orm sqlite_orm_ep) +endmacro() + +if(MEGASEARCH_WITH_SQLITE_ORM) + resolve_dependency(SQLite_ORM) +# ExternalProject_Get_Property(sqlite_orm_ep source_dir) +# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep) + include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") + #message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}") +endif() + +# ---------------------------------------------------------------------- +# Thrift + +macro(build_thrift) + message(STATUS "Building Apache Thrift-${THRIFT_VERSION} from source") + set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep") + set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") + set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") + set(THRIFT_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" + "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DBUILD_EXAMPLES=OFF + -DBUILD_TUTORIALS=OFF + -DWITH_QT4=OFF + -DWITH_C_GLIB=OFF + -DWITH_JAVA=OFF + -DWITH_PYTHON=OFF + -DWITH_HASKELL=OFF + -DWITH_CPP=ON + -DWITH_STATIC_LIB=ON + -DWITH_LIBEVENT=OFF + -DCMAKE_BUILD_TYPE=Release) + + # Thrift also uses boost. Forward important boost settings if there were ones passed. + if(DEFINED BOOST_ROOT) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}") + endif() + if(DEFINED Boost_NAMESPACE) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}") + endif() + + set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") + if(MSVC) + if(MEGASEARCH_USE_STATIC_CRT) + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") + else() + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") + endif() + endif() + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") + endif() + set(THRIFT_STATIC_LIB + "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + if(ZLIB_SHARED_LIB) + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" ${THRIFT_CMAKE_ARGS}) + else() + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" ${THRIFT_CMAKE_ARGS}) + endif() + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) + + if(MSVC) + set(WINFLEXBISON_VERSION 2.4.9) + set(WINFLEXBISON_PREFIX + "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") + externalproject_add( + winflexbison_ep + URL + https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip + URL_HASH + MD5=a2e979ea9928fbf8567e995e9c0df765 + SOURCE_DIR + ${WINFLEXBISON_PREFIX} + CONFIGURE_COMMAND + "" + BUILD_COMMAND + "" + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) + + set(THRIFT_CMAKE_ARGS + "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" + "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" + "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" + "-DWITH_SHARED_LIB=OFF" + "-DWITH_PLUGIN=OFF" + ${THRIFT_CMAKE_ARGS}) + elseif(APPLE) + # Some other process always resets BISON_EXECUTABLE to the system default, + # thus we use our own variable here. + if(NOT DEFINED THRIFT_BISON_EXECUTABLE) + find_package(BISON 2.5.1) + + # In the case where we cannot find a system-wide installation, look for + # homebrew and ask for its bison installation. + if(NOT BISON_FOUND) + find_program(BREW_BIN brew) + if(BREW_BIN) + execute_process(COMMAND ${BREW_BIN} --prefix bison + OUTPUT_VARIABLE BISON_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") + find_package(BISON 2.5.1) + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + else() + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + endif() + set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" + ${THRIFT_CMAKE_ARGS}) + endif() + + externalproject_add(thrift_ep + URL + ${THRIFT_SOURCE_URL} + BUILD_BYPRODUCTS + "${THRIFT_STATIC_LIB}" + "${THRIFT_COMPILER}" + CMAKE_ARGS + ${THRIFT_CMAKE_ARGS} + DEPENDS + ${THRIFT_DEPENDENCIES} + ${EP_LOG_OPTIONS}) + + add_library(thrift STATIC IMPORTED) + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${THRIFT_INCLUDE_DIR}") + set_target_properties(thrift + PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") + add_dependencies(thrift thrift_ep) +endmacro() + +if(MEGASEARCH_WITH_THRIFT) + resolve_dependency(Thrift) + # TODO: Don't use global includes but rather target_include_directories +# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/) + link_directories(SYSTEM ${THRIFT_PREFIX}/lib/) + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# yaml-cpp + +macro(build_yamlcpp) + message(STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source") + set(YAMLCPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix/src/yaml-cpp_ep") + set(YAMLCPP_STATIC_LIB "${YAMLCPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(YAMLCPP_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${YAMLCPP_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -DYAML_CPP_BUILD_TESTS=OFF + -DYAML_CPP_BUILD_TOOLS=OFF) + + externalproject_add(yaml-cpp_ep + URL + ${YAMLCPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_BYPRODUCTS + "${YAMLCPP_STATIC_LIB}" + CMAKE_ARGS + ${YAMLCPP_CMAKE_ARGS}) - if(GTEST_VENDORED) - add_dependencies(gtest googletest) - add_dependencies(gtest_main googletest) + file(MAKE_DIRECTORY "${YAMLCPP_PREFIX}/include") + + add_library(yaml-cpp STATIC IMPORTED) + set_target_properties(yaml-cpp + PROPERTIES IMPORTED_LOCATION "${YAMLCPP_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${YAMLCPP_PREFIX}/include") + + add_dependencies(yaml-cpp yaml-cpp_ep) +endmacro() + +if(MEGASEARCH_WITH_YAMLCPP) + resolve_dependency(yaml-cpp) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) + include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# zlib + +macro(build_zlib) + message(STATUS "Building ZLIB-${ZLIB_VERSION} from source") + set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep") + if(MSVC) + if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) + else() + set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) + endif() + else() + set(ZLIB_STATIC_LIB_NAME libz.a) endif() -endif() \ No newline at end of file + set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") + set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) + + externalproject_add(zlib_ep + URL + ${ZLIB_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_BYPRODUCTS + "${ZLIB_STATIC_LIB}" + CMAKE_ARGS + ${ZLIB_CMAKE_ARGS}) + + file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include") + + add_library(zlib STATIC IMPORTED) + set_target_properties(zlib + PROPERTIES IMPORTED_LOCATION "${ZLIB_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include") + + add_dependencies(zlib zlib_ep) +endmacro() + +if(MEGASEARCH_WITH_ZLIB) + resolve_dependency(ZLIB) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# zstd + +macro(build_zstd) + message(STATUS "Building zstd-${ZSTD_VERSION} from source") + set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep") + + set(ZSTD_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DZSTD_BUILD_PROGRAMS=off + -DZSTD_BUILD_SHARED=off + -DZSTD_BUILD_STATIC=on + -DZSTD_MULTITHREAD_SUPPORT=off) + + if(MSVC) + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib") + if(MEGASEARCH_USE_STATIC_CRT) + set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") + endif() + else() + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") + # Only pass our C flags on Unix as on MSVC it leads to a + # "incompatible command-line options" error + set(ZSTD_CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) + endif() + + if(CMAKE_VERSION VERSION_LESS 3.7) + message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") + endif() + + externalproject_add(zstd_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + SOURCE_SUBDIR + "build/cmake" + INSTALL_DIR + ${ZSTD_PREFIX} + URL + ${ZSTD_SOURCE_URL} + BUILD_BYPRODUCTS + "${ZSTD_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ZSTD_PREFIX}/include") + + add_library(zstd STATIC IMPORTED) + set_target_properties(zstd + PROPERTIES IMPORTED_LOCATION "${ZSTD_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_PREFIX}/include") + + add_dependencies(zstd zstd_ep) +endmacro() + +if(MEGASEARCH_WITH_ZSTD) + resolve_dependency(ZSTD) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${ZSTD_PREFIX}/lib) + include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) +endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index f91944fae49eefcdd33ac13a0375f41cdc285007..40034d59d3745bedc5c65b32550e1a2276e50925 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -47,11 +47,52 @@ set(vecwise_engine_files set(get_sys_info_files license/GetSysInfo.cpp) - include_directories(/usr/include) include_directories(/usr/local/cuda/include) include_directories(thrift/gen-cpp) +#target_link_libraries(megasearch boost_system_static) +#target_link_libraries(megasearch boost_filesystem_static) +#target_link_libraries(megasearch boost_serialization_static) +#target_link_libraries(megasearch bzip2) +#target_link_libraries(megasearch easyloggingpp) +#target_link_libraries(megasearch faiss) +#target_link_libraries(megasearch gtest) +#target_link_libraries(megasearch lapack) +#target_link_libraries(megasearch lz4) +#target_link_libraries(megasearch openblas) +#target_link_libraries(megasearch rocksdb) +#target_link_libraries(megasearch snappy) +#target_link_libraries(megasearch sqlite) +#target_link_libraries(megasearch sqlite_orm) +#target_link_libraries(megasearch thrift) +#target_link_libraries(megasearch yaml-cpp) +#target_link_libraries(megasearch zlib) +#target_link_libraries(megasearch zstd) + +set(third_party_libs + boost_system_static + boost_filesystem_static + boost_serialization_static + bzip2 + easyloggingpp + faiss + gtest + gtest_main + gmock + lapack + lz4 + openblas + rocksdb + snappy + sqlite +# sqlite_orm + thrift + yaml-cpp + zlib + zstd + ) + if (GPU_VERSION STREQUAL "ON") link_directories(/usr/local/cuda/lib64) set(engine_libs @@ -100,13 +141,13 @@ endif () cuda_add_library(vecwise_engine STATIC ${vecwise_engine_files}) -target_link_libraries(vecwise_engine ${engine_libs}) +target_link_libraries(vecwise_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) if (ENABLE_LICENSE STREQUAL "ON") add_library(vecwise_license STATIC ${license_check_files}) - target_link_libraries(vecwise_license ${license_libs}) + target_link_libraries(vecwise_license ${license_libs} ${third_party_libs}) endif () #set(metrics_lib @@ -118,17 +159,6 @@ endif () #add_library(vecwise_engine STATIC ${metrics_files} ) #target_link_libraries(metrics ${metrics_lib}) -add_executable(vecwise_server - ${config_files} - ${server_files} - ${utils_files} - ${service_files} - ${metrics_files} - ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc - ) - - - set(server_libs vecwise_engine librocksdb.a @@ -145,16 +175,30 @@ set(server_libs dl ) +add_executable(vecwise_server + ${config_files} + ${server_files} + ${utils_files} + ${service_files} + ${metrics_files} + #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc + ) + if (ENABLE_LICENSE STREQUAL "ON") - target_link_libraries(vecwise_server ${server_libs} vecwise_license) + target_link_libraries(vecwise_server ${server_libs} vecwise_license ${third_party_libs}) else () - target_link_libraries(vecwise_server ${server_libs}) + target_link_libraries(vecwise_server ${server_libs} ${third_party_libs}) endif() if (ENABLE_LICENSE STREQUAL "ON") + add_executable(get_sys_info ${get_sys_info_files}) add_executable(license_generator ${license_generator_files}) - target_link_libraries(get_sys_info ${license_libs} vecwise_license) - target_link_libraries(license_generator ${license_libs}) + + target_link_libraries(get_sys_info ${license_libs} vecwise_license ${third_party_libs}) + target_link_libraries(license_generator ${license_libs} ${third_party_libs}) + + install(TARGETS get_sys_info DESTINATION bin) + install(TARGETS license_generator DESTINATION bin) endif () install(TARGETS vecwise_server DESTINATION bin) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index 600eecb4adc64c81628be5b0850b93a9677a1813..74f34d86153257f5f961334e433fdd9b6813db79 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -28,8 +28,7 @@ add_library(megasearch_sdk STATIC link_directories(../../third_party/build/lib) target_link_libraries(megasearch_sdk - libthrift.a - pthread + ${third_party_libs} ) add_subdirectory(examples) diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index a630442c987ad602e169d2951e69500c3b71dfac..8bb387bdd95d81fb9d660d93916517fc8c098878 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -8,7 +8,7 @@ #include "utils/Error.h" #include "config/ConfigNode.h" -#include +#include "yaml-cpp/yaml.h" namespace zilliz { namespace vecwise { diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt new file mode 100644 index 0000000000000000000000000000000000000000..772c70901601a5989c3b22c4fb9da8922430efc9 --- /dev/null +++ b/cpp/thirdparty/versions.txt @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Toolchain library versions +# +# This file is used by `download_dependencies.sh` and cmake to figure out which +# version of a dependency to fetch. In order to add a new dependency, add a +# version variable, e.g. MY_DEP_VERSION and append an entry in the +# `DEPENDENCIES` array (see the comment on top of the declaration for the +# format). + +BOOST_VERSION=1.70.0 +BZIP2_VERSION=1.0.6 +EASYLOGGINGPP_VERSION=v9.96.7 +FAISS_VERSION=master +GTEST_VERSION=1.8.1 +LAPACK_VERSION=v3.8.0 +LZ4_VERSION=v1.9.1 +OPENBLAS_VERSION=v0.3.6 +ROCKSDB_VERSION=v6.0.2 +SNAPPY_VERSION=1.1.7 +SQLITE_VERSION=3280000 +SQLITE_ORM_VERSION=master +THRIFT_VERSION=0.12.0 +YAMLCPP_VERSION=0.6.2 +ZLIB_VERSION=v1.2.11 +ZSTD_VERSION=v1.4.0 + +# The first field is the name of the environment variable expected by cmake. +# This _must_ match what is defined. The second field is the name of the +# generated archive file. The third field is the url of the project for the +# given version. +DEPENDENCIES=( + "MEGASEARCH_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz" + "MEGASEARCH_BZIP2_URL bzip2-${BZIP2_VERSION}.tar.gz https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz" + "MEGASEARCH_EASYLOGGINGPP_URL easyloggingpp-${EASYLOGGINGPP_VERSION}.tar.gz https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz" + "MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz" + "MEGASEARCH_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" + "MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz + "MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz" + "MEGASEARCH_OPENBLAS_URL openblas-${OPENBLAS_VERSION}.tar.gz https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz" + "MEGASEARCH_ROCKSDB_URL rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" + "MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" + "MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz" + "MEGASEARCH_SQLITE_ORM_URL sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz" + "MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" + "MEGASEARCH_YAMLCPP_URL yaml-cpp-${YAMLCPP_VERSION}.tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" + "MEGASEARCH_ZLIB_URL zlib-${ZLIB_VERSION}.tar.gz https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz" + "MEGASEARCH_ZSTD_URL zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz" + ) + +# vim: set filetype=sh: \ No newline at end of file diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 652fd4927a521d70282d718a6a49b8a60ca69d7a..7d370428962334efe217f5d380be4d6049f73d73 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -5,18 +5,26 @@ #------------------------------------------------------------------------------- link_directories( "${CMAKE_BINARY_DIR}/lib" - "${VECWISE_THIRD_PARTY_BUILD}/lib" + #"${VECWISE_THIRD_PARTY_BUILD}/lib" + "${GTEST_PREFIX}/lib/" ) +message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib") set(unittest_srcs - ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc) + ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp) + #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc) set(unittest_libs yaml-cpp + gtest + gmock gtest_main gmock_main + easyloggingpp pthread - metrics) + metrics + openblas + gfortran) add_subdirectory(server) add_subdirectory(db) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 30fa0f4a20cb5083668364e967dd3679daa357e7..2b3d4609b0ccc4d0f74fe1641cfbc91ac0e68d2e 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -42,4 +42,4 @@ set(db_libs lz4 ) -target_link_libraries(db_test ${unittest_libs} ${db_libs}) +target_link_libraries(db_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index eb179e6b6a5c8c6c6008c0ccbb29e0bf1c7f0b02..1ea5431586b1c2e47d5e344764a8d3d3491ffcbe 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -45,7 +45,7 @@ set(wrapper_libs zstd lz4 ) -target_link_libraries(wrapper_test ${unittest_libs} ${wrapper_libs}) +target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) set(topk_test_src topk_test.cpp diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt index 55509c195c66add2871e1e71acb947850a069b5b..1182d256fcf92581bc7bfdae9bba5d3c8f6cf8f1 100644 --- a/cpp/unittest/license/CMakeLists.txt +++ b/cpp/unittest/license/CMakeLists.txt @@ -16,6 +16,7 @@ link_directories(/usr/local/cuda/lib64) link_directories(/usr/lib/x86_64-linux-gnu) link_directories(/usr/lib/nvidia-415) link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/) +link_directories(/usr/local/cuda/lib64/stubs/) set(require_files ../../src/license/LicenseLibrary.cpp @@ -44,4 +45,4 @@ set(db_libs boost_serialization ) -target_link_libraries(license_test ${unittest_libs} ${db_libs}) +target_link_libraries(license_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 25fed9669499cbecd225c9fd7d6b5acbd5e44115..954f34d141165583c540214975f891ff7847e7b0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -57,7 +57,7 @@ set(count_test_src add_executable(metrics_test ${count_test_src} ${require_files} ) target_link_libraries(metrics_test - ${unittest_libs} + faiss cudart cublas @@ -72,5 +72,5 @@ target_link_libraries(metrics_test # prometheus-cpp-core pthread z - + ${unittest_libs} ) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 5d081d72c9d1de368e27c7ab898650f1282b82ea..f4ffe07fa85b38b5ebe4c82c8160c9129fab6940 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -51,7 +51,7 @@ set(require_libs ) target_link_libraries(server_test - ${unittest_libs} ${require_libs} ${cuda_library} + ${unittest_libs} )