Merge remote-tracking branch 'main/branch-1.2' into branch-1.2

Former-commit-id: 3ec3ca22cca08e517779bc268c535d5edd8768fe

cpp/CHANGELOG.md

@@ -12,9 +12,14 @@ Please mark all change in change log and use the ticket from JIRA.
 
 - MS-5 - Implement Auto Archive Feature
 - MS-16 - Implement metrics without prometheus
+- MS-6 - Implement SDK interface part 1
+- MS-21 - Implement SDK interface part 2
+- MS-26 - cmake. Add thirdparty packages
+- MS-31 - cmake: add prometheus
+
 ### Task
 
 - MS-1 - Add CHANGELOG.md
 - MS-4 - Refactor the vecwise_engine code structure
-- MS-6 - Implement SDK interface part 1
 - MS-20 - Clean Code Part 1
+- MS-30 - Use faiss v1.5.2

cpp/CMakeLists.txt

@@ -4,10 +4,33 @@
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
 
-cmake_minimum_required(VERSION 3.12)
+cmake_minimum_required(VERSION 3.14)
+message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
+set(MEGASEARCH_VERSION "0.1.0")
+
+string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MEGASEARCH_BASE_VERSION "${MEGASEARCH_VERSION}")
+
+project(megasearch VERSION "${MEGASEARCH_BASE_VERSION}")
 project(vecwise_engine LANGUAGES CUDA CXX)
 
+set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}")
+set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}")
+set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}")
+
+if(MEGASEARCH_VERSION_MAJOR STREQUAL ""
+        OR MEGASEARCH_VERSION_MINOR STREQUAL ""
+        OR MEGASEARCH_VERSION_PATCH STREQUAL "")
+    message(FATAL_ERROR "Failed to determine MegaSearch version from '${MEGASEARCH_VERSION}'")
+endif()
+
+message(STATUS "MegaSearch version: "
+        "${MEGASEARCH_VERSION_MAJOR}.${MEGASEARCH_VERSION_MINOR}.${MEGASEARCH_VERSION_PATCH} "
+        "(full: '${MEGASEARCH_VERSION}')")
+
+set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR})
+set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR})
+
 find_package(CUDA)
 set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
 set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
@@ -19,6 +42,7 @@ if (GPU_VERSION STREQUAL "ON")
 endif ()
 
 set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED on)
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
     message("building vecwise_engine on x86 architecture")
@@ -32,49 +56,52 @@ else()
     set(VECWISE_BUILD_ARCH unknown)
 endif()
 
-if(DEFINED UNIX)
-    message("building vecwise on Unix")
-    set(VECWISE_BUILD_SYSTEM macos)
-elseif(DEFINED APPLE)
-    message("building vecwise on MacOS")
-    set(VECWISE_BUILD_SYSTEM unix)
-else()
-    message("unknown OS")
-    set(VECWISE_BUILD_SYSTEM unknown)
-endif ()
-
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
     if (GPU_VERSION STREQUAL "ON")
         set(ENABLE_LICENSE "ON")
         add_definitions("-DENABLE_LICENSE")
     endif ()
 else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
 endif()
 
-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/" ${CMAKE_MODULE_PATH})
+
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+#if (BUILD_UNIT_TEST)
+#    option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON)
+#endif(BUILD_UNIT_TEST)
+
+include(ExternalProject)
+include(DefineOptions)
+include(BuildUtils)
+include(ThirdPartyPackages)
+
+include_directories(${MEGASEARCH_SOURCE_DIR})
+link_directories(${MEGASEARCH_BINARY_DIR})
+
+## Following should be check
 
 set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
 set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
-set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
-set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
+#set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
+#set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
 
 add_compile_definitions(PROFILER=${PROFILER})
 
 include_directories(${VECWISE_ENGINE_INCLUDE})
 include_directories(${VECWISE_ENGINE_SRC})
-include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
+#include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
 
 link_directories(${CMAKE_CURRRENT_BINARY_DIR})
-link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
-link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
+#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
+#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
 #execute_process(COMMAND bash build.sh
 #                WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
 
 add_subdirectory(src)
-add_subdirectory(unittest)
-
 
 if (BUILD_UNIT_TEST)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
@@ -93,3 +120,5 @@ install(FILES
         conf/vecwise_engine_log.conf
         DESTINATION
         conf)
+
+config_summary()

cpp/build.sh

@@ -72,7 +72,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then
     make clean
 fi
 
-make -j || exit 1
+make -j 1 || exit 1
 
 if [[ ${BUILD_TYPE} != "Debug" ]]; then
     strip src/vecwise_server

cpp/cmake/BuildUtils.cmake

+function(ADD_THIRDPARTY_LIB LIB_NAME)
+    set(options)
+    set(one_value_args SHARED_LIB STATIC_LIB)
+    set(multi_value_args DEPS INCLUDE_DIRECTORIES)
+    cmake_parse_arguments(ARG
+            "${options}"
+            "${one_value_args}"
+            "${multi_value_args}"
+            ${ARGN})
+    if(ARG_UNPARSED_ARGUMENTS)
+        message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+    endif()
+
+    if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
+        if(NOT ARG_STATIC_LIB)
+            message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+        endif()
+
+        set(AUG_LIB_NAME "${LIB_NAME}_static")
+        add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+        set_target_properties(${AUG_LIB_NAME}
+                PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+        if(ARG_DEPS)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
+        endif()
+        message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
+        if(ARG_INCLUDE_DIRECTORIES)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                    "${ARG_INCLUDE_DIRECTORIES}")
+        endif()
+
+        set(AUG_LIB_NAME "${LIB_NAME}_shared")
+        add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+
+        if(WIN32)
+            # Mark the ".lib" location as part of a Windows DLL
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+        else()
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+        endif()
+        if(ARG_DEPS)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
+        endif()
+        message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
+        if(ARG_INCLUDE_DIRECTORIES)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                    "${ARG_INCLUDE_DIRECTORIES}")
+        endif()
+    elseif(ARG_STATIC_LIB)
+        set(AUG_LIB_NAME "${LIB_NAME}_static")
+        add_library(${AUG_LIB_NAME} STATIC IMPORTED)
+        set_target_properties(${AUG_LIB_NAME}
+                PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+        if(ARG_DEPS)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
+        endif()
+        message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
+        if(ARG_INCLUDE_DIRECTORIES)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                    "${ARG_INCLUDE_DIRECTORIES}")
+        endif()
+    elseif(ARG_SHARED_LIB)
+        set(AUG_LIB_NAME "${LIB_NAME}_shared")
+        add_library(${AUG_LIB_NAME} SHARED IMPORTED)
+
+        if(WIN32)
+            # Mark the ".lib" location as part of a Windows DLL
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
+        else()
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
+        endif()
+        message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
+        if(ARG_DEPS)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
+        endif()
+        if(ARG_INCLUDE_DIRECTORIES)
+            set_target_properties(${AUG_LIB_NAME}
+                    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                    "${ARG_INCLUDE_DIRECTORIES}")
+        endif()
+    else()
+        message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
+    endif()
+endfunction()
\ No newline at end of file

cpp/cmake/DefineOptions.cmake

+
+macro(set_option_category name)
+    set(MEGASEARCH_OPTION_CATEGORY ${name})
+    list(APPEND "MEGASEARCH_OPTION_CATEGORIES" ${name})
+endmacro()
+
+macro(define_option name description default)
+    option(${name} ${description} ${default})
+    list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
+    set("${name}_OPTION_DESCRIPTION" ${description})
+    set("${name}_OPTION_DEFAULT" ${default})
+    set("${name}_OPTION_TYPE" "bool")
+endmacro()
+
+function(list_join lst glue out)
+    if("${${lst}}" STREQUAL "")
+        set(${out} "" PARENT_SCOPE)
+        return()
+    endif()
+
+    list(GET ${lst} 0 joined)
+    list(REMOVE_AT ${lst} 0)
+    foreach(item ${${lst}})
+        set(joined "${joined}${glue}${item}")
+    endforeach()
+    set(${out} ${joined} PARENT_SCOPE)
+endfunction()
+
+macro(define_option_string name description default)
+    set(${name} ${default} CACHE STRING ${description})
+    list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
+    set("${name}_OPTION_DESCRIPTION" ${description})
+    set("${name}_OPTION_DEFAULT" "\"${default}\"")
+    set("${name}_OPTION_TYPE" "string")
+
+    set("${name}_OPTION_ENUM" ${ARGN})
+    list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
+    if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+        set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
+    endif()
+endmacro()
+
+#----------------------------------------------------------------------
+set_option_category("Thirdparty")
+
+set(MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+
+define_option_string(MEGASEARCH_DEPENDENCY_SOURCE
+                    "Method to use for acquiring MEGASEARCH's build dependencies"
+                    "${MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT}"
+                    "AUTO"
+                    "BUNDLED"
+                    "SYSTEM")
+
+define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD
+        "Show output from ExternalProjects rather than just logging to files" ON)
+
+define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF)
+
+define_option(MEGASEARCH_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
+Note that this requires linking Boost statically" ON)
+
+define_option(MEGASEARCH_BOOST_HEADER_ONLY "Use only BOOST headers" OFF)
+
+define_option(MEGASEARCH_WITH_BZ2 "Build with BZ2 compression" ON)
+
+define_option(MEGASEARCH_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
+
+define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON)
+
+define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
+
+define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against"
+        "-gencode=arch=compute_61,code=sm_61")
+
+define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON)
+
+define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON)
+
+define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON)
+
+define_option(MEGASEARCH_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
+
+define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON)
+
+define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON)
+
+define_option(MEGASEARCH_WITH_SQLITE "Build with SQLite library" ON)
+
+define_option(MEGASEARCH_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
+
+define_option(MEGASEARCH_WITH_THRIFT "Build with Apache Thrift library" ON)
+
+define_option(MEGASEARCH_WITH_YAMLCPP "Build with yaml-cpp library" ON)
+
+define_option(MEGASEARCH_WITH_ZLIB "Build with zlib compression" ON)
+
+if(CMAKE_VERSION VERSION_LESS 3.7)
+    set(MEGASEARCH_WITH_ZSTD_DEFAULT OFF)
+else()
+    # ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7.
+    set(MEGASEARCH_WITH_ZSTD_DEFAULT ON)
+endif()
+define_option(MEGASEARCH_WITH_ZSTD "Build with zstd compression" ${MEGASEARCH_WITH_ZSTD_DEFAULT})
+
+#----------------------------------------------------------------------
+if(MSVC)
+    set_option_category("MSVC")
+
+    define_option(MSVC_LINK_VERBOSE
+            "Pass verbose linking options when linking libraries and executables"
+            OFF)
+
+    define_option(MEGASEARCH_USE_STATIC_CRT "Build MEGASEARCH with statically linked CRT" OFF)
+endif()
+
+
+#----------------------------------------------------------------------
+set_option_category("Test and benchmark")
+
+if (BUILD_UNIT_TEST)
+    define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON)
+else()
+    define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" OFF)
+endif(BUILD_UNIT_TEST)
+
+#----------------------------------------------------------------------
+macro(config_summary)
+    message(STATUS "---------------------------------------------------------------------")
+    message(STATUS "MEGASEARCH version:                                 ${MEGASEARCH_VERSION}")
+    message(STATUS)
+    message(STATUS "Build configuration summary:")
+
+    message(STATUS "  Generator: ${CMAKE_GENERATOR}")
+    message(STATUS "  Build type: ${CMAKE_BUILD_TYPE}")
+    message(STATUS "  Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
+    if(${CMAKE_EXPORT_COMPILE_COMMANDS})
+        message(
+                STATUS "  Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
+    endif()
+
+    foreach(category ${MEGASEARCH_OPTION_CATEGORIES})
+
+        message(STATUS)
+        message(STATUS "${category} options:")
+
+        set(option_names ${MEGASEARCH_${category}_OPTION_NAMES})
+
+        set(max_value_length 0)
+        foreach(name ${option_names})
+            string(LENGTH "\"${${name}}\"" value_length)
+            if(${max_value_length} LESS ${value_length})
+                set(max_value_length ${value_length})
+            endif()
+        endforeach()
+
+        foreach(name ${option_names})
+            if("${${name}_OPTION_TYPE}" STREQUAL "string")
+                set(value "\"${${name}}\"")
+            else()
+                set(value "${${name}}")
+            endif()
+
+            set(default ${${name}_OPTION_DEFAULT})
+            set(description ${${name}_OPTION_DESCRIPTION})
+            string(LENGTH ${description} description_length)
+            if(${description_length} LESS 70)
+                string(
+                        SUBSTRING
+                        "                                                                     "
+                        ${description_length} -1 description_padding)
+            else()
+                set(description_padding "
+                ")
+            endif()
+
+            set(comment "[${name}]")
+
+            if("${value}" STREQUAL "${default}")
+                set(comment "[default] ${comment}")
+            endif()
+
+            if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
+                set(comment "${comment} [${${name}_OPTION_ENUM}]")
+            endif()
+
+            string(
+                    SUBSTRING "${value}                                                             "
+                    0 ${max_value_length} value)
+
+            message(STATUS "  ${description} ${description_padding} ${value} ${comment}")
+        endforeach()
+
+    endforeach()
+
+endmacro()

cpp/cmake/GoogleTest.cmake

-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tries to find GTest headers and libraries.
-#
-# Usage of this module as follows:
-#
-#  find_package(GTest)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-#  GTest_HOME - When set, this path is inspected instead of standard library
-#                locations as the root of the GTest installation.
-#                The environment variable GTEST_HOME overrides this veriable.
-#
-# This module defines
-#  GTEST_INCLUDE_DIR, directory containing headers
-#  GTEST_LIBS, directory containing gtest libraries
-#  GTEST_STATIC_LIB, path to libgtest.a
-#  GTEST_SHARED_LIB, path to libgtest's shared library
-#  GTEST_FOUND, whether gtest has been found
-
-if( NOT "${GTEST_HOME}" STREQUAL "")
-    file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path )
-    list( APPEND _gtest_roots ${_native_path} )
-elseif ( GTest_HOME )
-    list( APPEND _gtest_roots ${GTest_HOME} )
-endif()
-
-# Try the parameterized roots, if they exist
-if ( _gtest_roots )
-    find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
-            PATHS ${_gtest_roots} NO_DEFAULT_PATH
-            PATH_SUFFIXES "include" )
-    find_library( GTEST_LIBRARIES NAMES gtest gtest_main
-            PATHS ${_gtest_roots} NO_DEFAULT_PATH
-            PATH_SUFFIXES "lib" )
-else ()
-    find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h )
-    find_library( GTEST_LIBRARIES NAMES gtest )
-endif ()
-
-
-if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
-    set(GTEST_FOUND TRUE)
-    get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
-    set(GTEST_LIB_NAME gtest)
-    set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
-    set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX})
-    set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
-else ()
-    set(GTEST_FOUND FALSE)
-endif ()
-
-if (GTEST_FOUND)
-    if (NOT GTest_FIND_QUIETLY)
-        message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}")
-    endif ()
-else ()
-    if (NOT GTest_FIND_QUIETLY)
-        set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ")
-        if ( _gtest_roots )
-            set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.")
-        else ()
-            set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.")
-        endif ()
-        if (GTest_FIND_REQUIRED)
-            message(FATAL_ERROR "${GTEST_ERR_MSG}")
-        else (GTest_FIND_REQUIRED)
-            message(STATUS "${GTEST_ERR_MSG}")
-        endif (GTest_FIND_REQUIRED)
-    endif ()
-endif ()
-
-mark_as_advanced(
-        GTEST_INCLUDE_DIR
-        GTEST_LIBS
-        GTEST_LIBRARIES
-        GTEST_STATIC_LIB
-        GTEST_SHARED_LIB
-)

cpp/conf/server_config.yaml

 server_config:
   address: 0.0.0.0
   port: 33001
-  transfer_protocol: json     #optional: binary, compact, json, debug
+  transfer_protocol: binary   #optional: binary, compact, json
   server_mode: thread_pool    #optional: simple, thread_pool
   gpu_index: 0                #which gpu to be used
 

cpp/conf/server_config_template.yaml

 server_config:
   address: 0.0.0.0
   port: 33001
-  transfer_protocol: json     #optional: binary, compact, json, debug
+  transfer_protocol: binary    #optional: binary, compact, json
   server_mode: thread_pool    #optional: simple, thread_pool
   gpu_index: 0                #which gpu to be used
 

cpp/src/CMakeLists.txt

@@ -47,11 +47,52 @@ set(vecwise_engine_files
 set(get_sys_info_files
         license/GetSysInfo.cpp)
 
-
 include_directories(/usr/include)
 include_directories(/usr/local/cuda/include)
 include_directories(thrift/gen-cpp)
 
+#target_link_libraries(megasearch boost_system_static)
+#target_link_libraries(megasearch boost_filesystem_static)
+#target_link_libraries(megasearch boost_serialization_static)
+#target_link_libraries(megasearch bzip2)
+#target_link_libraries(megasearch easyloggingpp)
+#target_link_libraries(megasearch faiss)
+#target_link_libraries(megasearch gtest)
+#target_link_libraries(megasearch lapack)
+#target_link_libraries(megasearch lz4)
+#target_link_libraries(megasearch openblas)
+#target_link_libraries(megasearch rocksdb)
+#target_link_libraries(megasearch snappy)
+#target_link_libraries(megasearch sqlite)
+#target_link_libraries(megasearch sqlite_orm)
+#target_link_libraries(megasearch thrift)
+#target_link_libraries(megasearch yaml-cpp)
+#target_link_libraries(megasearch zlib)
+#target_link_libraries(megasearch zstd)
+
+set(third_party_libs
+        easyloggingpp
+        sqlite
+        #        sqlite_orm
+        thrift
+        yaml-cpp
+        faiss
+        lapack
+        openblas
+        prometheus-cpp-push
+        prometheus-cpp-pull
+        prometheus-cpp-core
+        rocksdb
+        boost_system_static
+        boost_filesystem_static
+        boost_serialization_static
+        bzip2
+        lz4
+        snappy
+        zlib
+        zstd
+        )
+
 if (GPU_VERSION STREQUAL "ON")
     link_directories(/usr/local/cuda/lib64)
     set(engine_libs
@@ -100,13 +141,13 @@ endif ()
 
 
 cuda_add_library(vecwise_engine STATIC ${vecwise_engine_files})
-target_link_libraries(vecwise_engine ${engine_libs})
+target_link_libraries(vecwise_engine ${engine_libs} ${third_party_libs})
 
 add_library(metrics STATIC ${metrics_files})
 
 if (ENABLE_LICENSE STREQUAL "ON")
         add_library(vecwise_license STATIC ${license_check_files})
-        target_link_libraries(vecwise_license ${license_libs})
+        target_link_libraries(vecwise_license ${license_libs} ${third_party_libs})
 endif ()
 
 #set(metrics_lib
@@ -118,17 +159,6 @@ endif ()
 #add_library(vecwise_engine STATIC ${metrics_files} )
 #target_link_libraries(metrics ${metrics_lib})
 
-add_executable(vecwise_server
-        ${config_files}
-        ${server_files}
-        ${utils_files}
-        ${service_files}
-        ${metrics_files}
-        ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc
-        )
-
-
-
 set(server_libs
         vecwise_engine
         librocksdb.a
@@ -146,16 +176,30 @@ set(server_libs
         metrics
         )
 
+add_executable(vecwise_server
+        ${config_files}
+        ${server_files}
+        ${utils_files}
+        ${service_files}
+        ${metrics_files}
+        #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc
+        )
+
 if (ENABLE_LICENSE STREQUAL "ON")
-        target_link_libraries(vecwise_server ${server_libs} vecwise_license)
+        target_link_libraries(vecwise_server ${server_libs} vecwise_license ${third_party_libs})
 else ()
-        target_link_libraries(vecwise_server ${server_libs})
+        target_link_libraries(vecwise_server ${server_libs} ${third_party_libs})
 endif()
 
 if (ENABLE_LICENSE STREQUAL "ON")
+        add_executable(get_sys_info ${get_sys_info_files})
         add_executable(license_generator ${license_generator_files})
-        target_link_libraries(get_sys_info ${license_libs} vecwise_license)
-        target_link_libraries(license_generator ${license_libs})
+
+        target_link_libraries(get_sys_info ${license_libs} vecwise_license ${third_party_libs})
+        target_link_libraries(license_generator ${license_libs} ${third_party_libs})
+
+        install(TARGETS get_sys_info DESTINATION bin)
+        install(TARGETS license_generator DESTINATION bin)
 endif ()
 
 install(TARGETS vecwise_server DESTINATION bin)

cpp/src/sdk/CMakeLists.txt

@@ -28,8 +28,7 @@ add_library(megasearch_sdk STATIC
 
 link_directories(../../third_party/build/lib)
 target_link_libraries(megasearch_sdk
-        libthrift.a
-        pthread
+        ${third_party_libs}
         )
 
 add_subdirectory(examples)

cpp/src/sdk/examples/simple/CMakeLists.txt

@@ -8,14 +8,12 @@ aux_source_directory(src src_files)
 
 include_directories(src)
 include_directories(../../megasearch_sdk/include)
-include_directories(/usr/include)
 
 link_directories(${CMAKE_BINARY_DIR}/megasearch_sdk)
 
 add_executable(sdk_simple
         ./main.cpp
         ${src_files}
-        ${service_files}
         )
 
 target_link_libraries(sdk_simple

cpp/src/sdk/examples/simple/main.cpp

@@ -19,7 +19,8 @@ main(int argc, char *argv[]) {
     printf("Client start...\n");
 
     std::string app_name = basename(argv[0]);
-    static struct option long_options[] = {{"conf_file", optional_argument, 0, 'c'},
+    static struct option long_options[] = {{"server", optional_argument, 0, 's'},
+                                           {"port", optional_argument, 0, 'p'},
                                            {"help", no_argument, 0, 'h'},
                                            {NULL, 0, 0, 0}};
 
@@ -28,9 +29,9 @@ main(int argc, char *argv[]) {
     app_name = argv[0];
 
     int value;
-    while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) {
+    while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) {
         switch (value) {
-            case 'h': {
+            case 's': {
                 char *address_ptr = strdup(optarg);
                 address = address_ptr;
                 free(address_ptr);
@@ -38,12 +39,14 @@ main(int argc, char *argv[]) {
             }
             case 'p': {
                 char *port_ptr = strdup(optarg);
-                address = port_ptr;
+                port = port_ptr;
                 free(port_ptr);
                 break;
             }
+            case 'h':
             default:
-                break;
+                print_help(app_name);
+                return EXIT_SUCCESS;
         }
     }
 
@@ -58,7 +61,8 @@ void
 print_help(const std::string &app_name) {
     printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
     printf("  Options:\n");
-    printf("   -h    Megasearch server address\n");
-    printf("   -p    Megasearch server port\n");
+    printf("   -s --server   Server address, default 127.0.0.1\n");
+    printf("   -p --port     Server port, default 33001\n");
+    printf("   -h --help     Print help information\n");
     printf("\n");
 }
\ No newline at end of file

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -13,13 +13,59 @@
 using namespace megasearch;
 
 namespace {
+    std::string GetTableName();
+
+    static const std::string TABLE_NAME = GetTableName();
+    static const std::string VECTOR_COLUMN_NAME = "face_vector";
+    static const std::string ID_COLUMN_NAME = "aid";
+    static const std::string CITY_COLUMN_NAME = "city";
+    static constexpr int64_t TABLE_DIMENSION = 512;
+    static constexpr int64_t TOTAL_ROW_COUNT = 100000;
+    static constexpr int64_t TOP_K = 10;
+    static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
+
+#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
+
     void PrintTableSchema(const megasearch::TableSchema& tb_schema) {
-        std::cout << "===========================================" << std::endl;
+        BLOCK_SPLITER
         std::cout << "Table name: " << tb_schema.table_name << std::endl;
         std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl;
         std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl;
         std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl;
-        std::cout << "===========================================" << std::endl;
+        BLOCK_SPLITER
+    }
+
+    void PrintRecordIdArray(const std::vector<int64_t>& record_ids) {
+        BLOCK_SPLITER
+        std::cout << "Returned id array count: " << record_ids.size() << std::endl;
+#if 0
+        for(auto id : record_ids) {
+            std::cout << std::to_string(id) << std::endl;
+        }
+#endif
+        BLOCK_SPLITER
+    }
+
+    void PrintSearchResult(const std::vector<TopKQueryResult>& topk_query_result_array) {
+        BLOCK_SPLITER
+        std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
+
+        int32_t index = 0;
+        for(auto& result : topk_query_result_array) {
+            index++;
+            std::cout << "No." << std::to_string(index) << " vector top "
+                << std::to_string(result.query_result_arrays.size())
+                << " search result:" << std::endl;
+            for(auto& item : result.query_result_arrays) {
+                std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score);
+                for(auto& attribute : item.column_map) {
+                    std::cout << "\t" << attribute.first << ":" << attribute.second;
+                }
+                std::cout << std::endl;
+            }
+        }
+
+        BLOCK_SPLITER
     }
 
     std::string CurrentTime() {
@@ -40,9 +86,24 @@ namespace {
         return s_id;
     }
 
-    static const std::string TABLE_NAME = GetTableName();
-    static const std::string VECTOR_COLUMN_NAME = "face_vector";
-    static const int64_t TABLE_DIMENSION = 512;
+    TableSchema BuildTableSchema() {
+        TableSchema tb_schema;
+        VectorColumn col1;
+        col1.name = VECTOR_COLUMN_NAME;
+        col1.dimension = TABLE_DIMENSION;
+        col1.store_raw_vector = true;
+        tb_schema.vector_column_array.emplace_back(col1);
+
+        Column col2 = {ColumnType::int8, ID_COLUMN_NAME};
+        tb_schema.attribute_column_array.emplace_back(col2);
+
+        Column col3 = {ColumnType::int16, CITY_COLUMN_NAME};
+        tb_schema.attribute_column_array.emplace_back(col3);
+
+        tb_schema.table_name = TABLE_NAME;
+
+        return tb_schema;
+    }
 
     void BuildVectors(int64_t from, int64_t to,
                       std::vector<RowRecord>* vector_record_array,
@@ -58,6 +119,19 @@ namespace {
             query_record_array->clear();
         }
 
+        static const std::map<int64_t , std::string> CITY_MAP = {
+                {0, "Beijing"},
+                {1, "Shanhai"},
+                {2, "Hangzhou"},
+                {3, "Guangzhou"},
+                {4, "Shenzheng"},
+                {5, "Wuhan"},
+                {6, "Chengdu"},
+                {7, "Chongqin"},
+                {8, "Tianjing"},
+                {9, "Hongkong"},
+        };
+
         for (int64_t k = from; k < to; k++) {
 
             std::vector<float> f_p;
@@ -69,12 +143,16 @@ namespace {
             if(vector_record_array) {
                 RowRecord record;
                 record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
+                record.attribute_map[ID_COLUMN_NAME] = std::to_string(k);
+                record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size());
                 vector_record_array->emplace_back(record);
             }
 
             if(query_record_array) {
                 QueryRecord record;
                 record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
+                record.selected_column_array.push_back(ID_COLUMN_NAME);
+                record.selected_column_array.push_back(CITY_COLUMN_NAME);
                 query_record_array->emplace_back(record);
             }
         }
@@ -84,61 +162,80 @@ namespace {
 void
 ClientTest::Test(const std::string& address, const std::string& port) {
     std::shared_ptr<Connection> conn = Connection::Create();
-    ConnectParam param = { address, port };
-    conn->Connect(param);
 
-    {//create table
-        TableSchema tb_schema;
-        VectorColumn col1;
-        col1.name = VECTOR_COLUMN_NAME;
-        col1.dimension = TABLE_DIMENSION;
-        col1.store_raw_vector = true;
-        tb_schema.vector_column_array.emplace_back(col1);
+    {//connect server
+        ConnectParam param = {address, port};
+        Status stat = conn->Connect(param);
+        std::cout << "Connect function call status: " << stat.ToString() << std::endl;
+    }
 
-        Column col2;
-        col2.name = "age";
-        tb_schema.attribute_column_array.emplace_back(col2);
+    {//server version
+        std::string version = conn->ServerVersion();
+        std::cout << "MegaSearch server version: " << version << std::endl;
+    }
 
-        tb_schema.table_name = TABLE_NAME;
+    {//sdk version
+        std::string version = conn->ClientVersion();
+        std::cout << "SDK version: " << version << std::endl;
+    }
+
+    {
+        std::vector<std::string> tables;
+        Status stat = conn->ShowTables(tables);
+        std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
+        std::cout << "All tables: " << std::endl;
+        for(auto& table : tables) {
+            std::cout << "\t" << table << std::endl;
+        }
+    }
 
+    {//create table
+        TableSchema tb_schema = BuildTableSchema();
         PrintTableSchema(tb_schema);
         Status stat = conn->CreateTable(tb_schema);
-        std::cout << "Create table result: " << stat.ToString() << std::endl;
+        std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
     }
 
     {//describe table
         TableSchema tb_schema;
         Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
-        std::cout << "Describe table result: " << stat.ToString() << std::endl;
+        std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
         PrintTableSchema(tb_schema);
     }
 
     {//add vectors
         std::vector<RowRecord> record_array;
-        BuildVectors(0, 10000, &record_array, nullptr);
+        BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr);
         std::vector<int64_t> record_ids;
-        std::cout << "Begin add vectors" << std::endl;
         Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
-        std::cout << "Add vector result: " << stat.ToString() << std::endl;
-        std::cout << "Returned vector ids: " << record_ids.size() << std::endl;
+        std::cout << "AddVector function call status: " << stat.ToString() << std::endl;
+        PrintRecordIdArray(record_ids);
     }
 
     {//search vectors
+        std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl;
         sleep(10);
         std::vector<QueryRecord> record_array;
-        BuildVectors(500, 510, nullptr, &record_array);
+        BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array);
 
         std::vector<TopKQueryResult> topk_query_result_array;
-        std::cout << "Begin search vectors" << std::endl;
-        Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, 10);
-        std::cout << "Search vector result: " << stat.ToString() << std::endl;
-        std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
+        Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K);
+        std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
+        PrintSearchResult(topk_query_result_array);
     }
 
 //    {//delete table
 //        Status stat = conn->DeleteTable(TABLE_NAME);
-//        std::cout << "Delete table result: " << stat.ToString() << std::endl;
+//        std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
 //    }
 
+    {//server status
+        std::string status = conn->ServerStatus();
+        std::cout << "Server status before disconnect: " << status << std::endl;
+    }
     Connection::Destroy(conn);
+    {//server status
+        std::string status = conn->ServerStatus();
+        std::cout << "Server status after disconnect: " << status << std::endl;
+    }
 }
\ No newline at end of file

cpp/src/sdk/include/Status.h

@@ -14,7 +14,8 @@ enum class StatusCode {
     OK = 0,
     Invalid = 1,
     UnknownError = 2,
-    NotSupported = 3
+    NotSupported = 3,
+    NotConnected = 4
 };
 
 /**

cpp/src/sdk/src/client/ClientProxy.cpp

@@ -16,32 +16,50 @@ ClientProxy::ClientPtr() const {
     return client_ptr;
 }
 
+bool ClientProxy::IsConnected() const {
+    return (client_ptr != nullptr && connected_);
+}
+
 Status
 ClientProxy::Connect(const ConnectParam &param) {
     Disconnect();
 
     int32_t port = atoi(param.port.c_str());
-    return ClientPtr()->Connect(param.ip_address, port, "json");
+    Status status = ClientPtr()->Connect(param.ip_address, port, THRIFT_PROTOCOL_BINARY);
+    if(status.ok()) {
+        connected_ = true;
+    }
+
+    return status;
 }
 
 Status
 ClientProxy::Connect(const std::string &uri) {
     Disconnect();
 
-    return Status::NotSupported("Connect interface is not supported.");
+    size_t index = uri.find_first_of(":", 0);
+    if((index == std::string::npos)) {
+        return Status::Invalid("Invalid uri");
+    }
+
+    ConnectParam param;
+    param.ip_address = uri.substr(0, index);
+    param.port = uri.substr(index + 1);
+
+    return Connect(param);
 }
 
 Status
 ClientProxy::Connected() const {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
         std::string info;
         ClientPtr()->interface()->Ping(info, "");
     }  catch ( std::exception& ex) {
-        return Status(StatusCode::UnknownError, "connection lost: " + std::string(ex.what()));
+        return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
     }
 
     return Status::OK();
@@ -49,22 +67,23 @@ ClientProxy::Connected() const {
 
 Status
 ClientProxy::Disconnect() {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
+    connected_ = false;
     return ClientPtr()->Disconnect();
 }
 
 std::string
 ClientProxy::ClientVersion() const {
-    return std::string("Current Version");
+    return std::string("v1.0");
 }
 
 Status
 ClientProxy::CreateTable(const TableSchema &param) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -103,8 +122,8 @@ ClientProxy::CreateTable(const TableSchema &param) {
 
 Status
 ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -132,8 +151,8 @@ ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
 
 Status
 ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -152,8 +171,8 @@ ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
 
 Status
 ClientProxy::DeleteTable(const std::string &table_name) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -170,8 +189,8 @@ Status
 ClientProxy::AddVector(const std::string &table_name,
                           const std::vector<RowRecord> &record_array,
                           std::vector<int64_t> &id_array) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -205,8 +224,8 @@ ClientProxy::SearchVector(const std::string &table_name,
                              const std::vector<QueryRecord> &query_record_array,
                              std::vector<TopKQueryResult> &topk_query_result_array,
                              int64_t topk) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -253,8 +272,8 @@ ClientProxy::SearchVector(const std::string &table_name,
 
 Status
 ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -290,8 +309,8 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
 
 Status
 ClientProxy::ShowTables(std::vector<std::string> &table_array) {
-    if(client_ptr == nullptr) {
-        return Status(StatusCode::UnknownError, "not connected");
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
     }
 
     try {
@@ -306,7 +325,7 @@ ClientProxy::ShowTables(std::vector<std::string> &table_array) {
 
 std::string
 ClientProxy::ServerVersion() const {
-    if(client_ptr == nullptr) {
+    if(!IsConnected()) {
         return "";
     }
 
@@ -321,8 +340,8 @@ ClientProxy::ServerVersion() const {
 
 std::string
 ClientProxy::ServerStatus() const {
-    if(client_ptr == nullptr) {
-        return "not connected";
+    if(!IsConnected()) {
+        return "not connected to server";
     }
 
     try {

cpp/src/sdk/src/client/ClientProxy.h

@@ -51,9 +51,11 @@ public:
 private:
     std::shared_ptr<ThriftClient>& ClientPtr() const;
 
+    bool IsConnected() const;
+
 private:
     mutable std::shared_ptr<ThriftClient> client_ptr;
-
+    bool connected_ = false;
 };
 
 }

cpp/src/sdk/src/client/ThriftClient.cpp

@@ -50,14 +50,12 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
         stdcxx::shared_ptr<TSocket> socket_ptr(new transport::TSocket(address, port));
         stdcxx::shared_ptr<TTransport> transport_ptr(new TBufferedTransport(socket_ptr));
         stdcxx::shared_ptr<TProtocol> protocol_ptr;
-        if(protocol == "binary") {
+        if(protocol == THRIFT_PROTOCOL_BINARY) {
             protocol_ptr.reset(new TBinaryProtocol(transport_ptr));
-        } else if(protocol == "json") {
+        } else if(protocol == THRIFT_PROTOCOL_JSON) {
             protocol_ptr.reset(new TJSONProtocol(transport_ptr));
-        } else if(protocol == "compact") {
+        } else if(protocol == THRIFT_PROTOCOL_COMPACT) {
             protocol_ptr.reset(new TCompactProtocol(transport_ptr));
-        } else if(protocol == "debug") {
-            protocol_ptr.reset(new TDebugProtocol(transport_ptr));
         } else {
             //CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently";
             return Status(StatusCode::Invalid, "unsupported protocol");
@@ -67,7 +65,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
         client_ = std::make_shared<thrift::MegasearchServiceClient>(protocol_ptr);
     } catch ( std::exception& ex) {
         //CLIENT_LOG_ERROR << "connect encounter exception: " << ex.what();
-        return Status(StatusCode::UnknownError, "failed to connect megasearch server" + std::string(ex.what()));
+        return Status(StatusCode::NotConnected, "failed to connect megasearch server" + std::string(ex.what()));
     }
 
     return Status::OK();

cpp/src/sdk/src/client/ThriftClient.h

@@ -14,6 +14,10 @@ namespace megasearch {
 
 using MegasearchServiceClientPtr = std::shared_ptr<megasearch::thrift::MegasearchServiceClient>;
 
+static const std::string THRIFT_PROTOCOL_JSON = "json";
+static const std::string THRIFT_PROTOCOL_BINARY = "binary";
+static const std::string THRIFT_PROTOCOL_COMPACT = "compact";
+
 class ThriftClient {
 public:
     ThriftClient();

cpp/src/sdk/src/interface/Status.cpp

@@ -101,6 +101,8 @@ std::string Status::CodeAsString() const {
             break;
         case StatusCode::NotSupported: type = "Not Supported";
             break;
+        case StatusCode::NotConnected: type = "Not Connected";
+            break;
         default: type = "Unknown";
             break;
     }

cpp/src/server/MegasearchHandler.cpp

@@ -32,14 +32,14 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) {
 
 void
 MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam &param) {
-    // Your implementation goes here
-    printf("CreateTablePartition\n");
+    BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param);
+    MegasearchScheduler::ExecTask(task_ptr);
 }
 
 void
 MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam &param) {
-    // Your implementation goes here
-    printf("DeleteTablePartition\n");
+    BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param);
+    MegasearchScheduler::ExecTask(task_ptr);
 }
 
 void
@@ -67,14 +67,14 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std:
 
 void
 MegasearchServiceHandler::ShowTables(std::vector<std::string> &_return) {
-    // Your implementation goes here
-    printf("ShowTables\n");
+    BaseTaskPtr task_ptr = ShowTablesTask::Create(_return);
+    MegasearchScheduler::ExecTask(task_ptr);
 }
 
 void
 MegasearchServiceHandler::Ping(std::string& _return, const std::string& cmd) {
-    // Your implementation goes here
-    printf("Ping\n");
+    BaseTaskPtr task_ptr = PingTask::Create(cmd, _return);
+    MegasearchScheduler::ExecTask(task_ptr);
 }
 
 }

cpp/src/server/MegasearchServer.cpp

@@ -54,7 +54,6 @@ MegasearchServer::StartService() {
         stdcxx::shared_ptr<TServerTransport> server_transport(new TServerSocket(address, port));
         stdcxx::shared_ptr<TTransportFactory> transport_factory(new TBufferedTransportFactory());
 
-        std::string protocol = "json";
         stdcxx::shared_ptr<TProtocolFactory> protocol_factory;
         if (protocol == "binary") {
             protocol_factory.reset(new TBinaryProtocolFactory());
@@ -62,8 +61,6 @@ MegasearchServer::StartService() {
             protocol_factory.reset(new TJSONProtocolFactory());
         } else if (protocol == "compact") {
             protocol_factory.reset(new TCompactProtocolFactory());
-        } else if (protocol == "debug") {
-            protocol_factory.reset(new TDebugProtocolFactory());
         } else {
             //SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently";
             return;

cpp/src/server/MegasearchTask.cpp

@@ -21,6 +21,7 @@ namespace server {
 
 static const std::string DQL_TASK_GROUP = "dql";
 static const std::string DDL_DML_TASK_GROUP = "ddl_dml";
+static const std::string PING_TASK_GROUP = "ping";
 
 static const std::string VECTOR_UID = "uid";
 static const uint64_t USE_MT = 5000;
@@ -48,6 +49,10 @@ namespace {
             }
         }
 
+        ~DBWrapper() {
+            delete db_;
+        }
+
         zilliz::vecwise::engine::DB* DB() { return db_; }
 
     private:
@@ -78,17 +83,17 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) {
 
 ServerError CreateTableTask::OnExecute() {
     TimeRecorder rc("CreateTableTask");
-
+    
     try {
         if(schema_.vector_column_array.empty()) {
             return SERVER_INVALID_ARGUMENT;
         }
 
         IVecIdMapper::GetInstance()->AddGroup(schema_.table_name);
-        engine::meta::TableSchema table_schema;
-        table_schema.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
-        table_schema.table_id = schema_.table_name;
-        engine::Status stat = DB()->CreateTable(table_schema);
+        engine::meta::TableSchema table_info;
+        table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
+        table_info.table_id = schema_.table_name;
+        engine::Status stat = DB()->CreateTable(table_info);
         if(!stat.ok()) {//could exist
             error_msg_ = "Engine failed: " + stat.ToString();
             SERVER_LOG_ERROR << error_msg_;
@@ -109,7 +114,7 @@ ServerError CreateTableTask::OnExecute() {
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 DescribeTableTask::DescribeTableTask(const std::string &table_name, thrift::TableSchema &schema)
-    : BaseTask(DDL_DML_TASK_GROUP),
+    : BaseTask(PING_TASK_GROUP),
       table_name_(table_name),
       schema_(schema) {
     schema_.table_name = table_name_;
@@ -123,9 +128,9 @@ ServerError DescribeTableTask::OnExecute() {
     TimeRecorder rc("DescribeTableTask");
 
     try {
-        engine::meta::TableSchema table_schema;
-        table_schema.table_id = table_name_;
-        engine::Status stat = DB()->DescribeTable(table_schema);
+        engine::meta::TableSchema table_info;
+        table_info.table_id = table_name_;
+        engine::Status stat = DB()->DescribeTable(table_info);
         if(!stat.ok()) {
             error_code_ = SERVER_GROUP_NOT_EXIST;
             error_msg_ = "Engine failed: " + stat.ToString();
@@ -154,8 +159,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name)
 
 }
 
-BaseTaskPtr DeleteTableTask::Create(const std::string& table_id) {
-    return std::shared_ptr<BaseTask>(new DeleteTableTask(table_id));
+BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) {
+    return std::shared_ptr<BaseTask>(new DeleteTableTask(group_id));
 }
 
 ServerError DeleteTableTask::OnExecute() {
@@ -168,6 +173,60 @@ ServerError DeleteTableTask::OnExecute() {
     return SERVER_NOT_IMPLEMENT;
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param)
+    : BaseTask(DDL_DML_TASK_GROUP),
+      param_(param) {
+
+}
+
+BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam &param) {
+    return std::shared_ptr<BaseTask>(new CreateTablePartitionTask(param));
+}
+
+ServerError CreateTablePartitionTask::OnExecute() {
+    error_code_ = SERVER_NOT_IMPLEMENT;
+    error_msg_ = "create table partition not implemented";
+    SERVER_LOG_ERROR << error_msg_;
+
+    return SERVER_NOT_IMPLEMENT;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param)
+    : BaseTask(DDL_DML_TASK_GROUP),
+      param_(param) {
+
+}
+
+BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam &param) {
+    return std::shared_ptr<BaseTask>(new DeleteTablePartitionTask(param));
+}
+
+ServerError DeleteTablePartitionTask::OnExecute() {
+    error_code_ = SERVER_NOT_IMPLEMENT;
+    error_msg_ = "delete table partition not implemented";
+    SERVER_LOG_ERROR << error_msg_;
+
+    return SERVER_NOT_IMPLEMENT;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+ShowTablesTask::ShowTablesTask(std::vector<std::string>& tables)
+    : BaseTask(PING_TASK_GROUP),
+      tables_(tables) {
+
+}
+
+BaseTaskPtr ShowTablesTask::Create(std::vector<std::string>& tables) {
+    return std::shared_ptr<BaseTask>(new ShowTablesTask(tables));
+}
+
+ServerError ShowTablesTask::OnExecute() {
+    IVecIdMapper::GetInstance()->AllGroups(tables_);
+
+    return SERVER_SUCCESS;
+}
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 AddVectorTask::AddVectorTask(const std::string& table_name,
@@ -178,7 +237,6 @@ AddVectorTask::AddVectorTask(const std::string& table_name,
       record_array_(record_array),
       record_ids_(record_ids) {
     record_ids_.clear();
-    record_ids_.resize(record_array.size());
 }
 
 BaseTaskPtr AddVectorTask::Create(const std::string& table_name,
@@ -195,9 +253,9 @@ ServerError AddVectorTask::OnExecute() {
             return SERVER_SUCCESS;
         }
 
-        engine::meta::TableSchema table_schema;
-        table_schema.table_id = table_name_;
-        engine::Status stat = DB()->DescribeTable(table_schema);
+        engine::meta::TableSchema table_info;
+        table_info.table_id = table_name_;
+        engine::Status stat = DB()->DescribeTable(table_info);
         if(!stat.ok()) {
             error_code_ = SERVER_GROUP_NOT_EXIST;
             error_msg_ = "Engine failed: " + stat.ToString();
@@ -208,7 +266,7 @@ ServerError AddVectorTask::OnExecute() {
         rc.Record("get group info");
 
         uint64_t vec_count = (uint64_t)record_array_.size();
-        uint64_t group_dim = table_schema.dimension;
+        uint64_t group_dim = table_info.dimension;
         std::vector<float> vec_f;
         vec_f.resize(vec_count*group_dim);//allocate enough memory
         for(uint64_t i = 0; i < vec_count; i++) {
@@ -228,6 +286,7 @@ ServerError AddVectorTask::OnExecute() {
                 return error_code_;
             }
 
+            //convert double array to float array(thrift has no float type)
             const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
             for(uint64_t d = 0; d < vec_dim; d++) {
                 vec_f[i*vec_dim + d] = (float)(d_p[d]);
@@ -245,12 +304,27 @@ ServerError AddVectorTask::OnExecute() {
             return error_code_;
         }
 
-        if(record_ids_.size() < vec_count) {
+        if(record_ids_.size() != vec_count) {
             SERVER_LOG_ERROR << "Vector ID not returned";
             return SERVER_UNEXPECTED_ERROR;
         }
 
-        rc.Record("done");
+        //persist attributes
+        for(uint64_t i = 0; i < vec_count; i++) {
+            const auto &record = record_array_[i];
+
+            //any attributes?
+            if(record.attribute_map.empty()) {
+                continue;
+            }
+
+            std::string nid = std::to_string(record_ids_[i]);
+            std::string attrib_str;
+            AttributeSerializer::Encode(record.attribute_map, attrib_str);
+            IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_);
+        }
+
+        rc.Record("persist vector attributes");
 
     } catch (std::exception& ex) {
         error_code_ = SERVER_UNEXPECTED_ERROR;
@@ -293,9 +367,9 @@ ServerError SearchVectorTask::OnExecute() {
             return error_code_;
         }
 
-        engine::meta::TableSchema table_schema;
-        table_schema.table_id = table_name_;
-        engine::Status stat = DB()->DescribeTable(table_schema);
+        engine::meta::TableSchema table_info;
+        table_info.table_id = table_name_;
+        engine::Status stat = DB()->DescribeTable(table_info);
         if(!stat.ok()) {
             error_code_ = SERVER_GROUP_NOT_EXIST;
             error_msg_ = "Engine failed: " + stat.ToString();
@@ -305,7 +379,7 @@ ServerError SearchVectorTask::OnExecute() {
 
         std::vector<float> vec_f;
         uint64_t record_count = (uint64_t)record_array_.size();
-        vec_f.resize(record_count*table_schema.dimension);
+        vec_f.resize(record_count*table_info.dimension);
 
         for(uint64_t i = 0; i < record_array_.size(); i++) {
             const auto& record = record_array_[i];
@@ -317,14 +391,15 @@ ServerError SearchVectorTask::OnExecute() {
             }
 
             uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value?
-            if (vec_dim != table_schema.dimension) {
+            if (vec_dim != table_info.dimension) {
                 SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
-                                 << " vs. group dimension:" << table_schema.dimension;
+                                 << " vs. group dimension:" << table_info.dimension;
                 error_code_ = SERVER_INVALID_VECTOR_DIMENSION;
                 error_msg_ = "Engine failed: " + stat.ToString();
                 return error_code_;
             }
 
+            //convert double array to float array(thrift has no float type)
             const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
             for(uint64_t d = 0; d < vec_dim; d++) {
                 vec_f[i*vec_dim + d] = (float)(d_p[d]);
@@ -336,25 +411,50 @@ ServerError SearchVectorTask::OnExecute() {
         std::vector<DB_DATE> dates;
         engine::QueryResults results;
         stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results);
+        rc.Record("search vectors from engine");
         if(!stat.ok()) {
             SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
             return SERVER_UNEXPECTED_ERROR;
-        } else {
-            rc.Record("do searching");
-            for(engine::QueryResult& result : results){
-                thrift::TopKQueryResult thrift_topk_result;
-                for(auto id : result) {
-                    thrift::QueryResult thrift_result;
-                    thrift_result.__set_id(id);
+        }
+
+        if(results.size() != record_count) {
+            SERVER_LOG_ERROR << "Search result not returned";
+            return SERVER_UNEXPECTED_ERROR;
+        }
+
+        //construct result array
+        for(uint64_t i = 0; i < record_count; i++) {
+            auto& result = results[i];
+            const auto& record = record_array_[i];
+
+            thrift::TopKQueryResult thrift_topk_result;
+            for(auto id : result) {
+                thrift::QueryResult thrift_result;
+                thrift_result.__set_id(id);
+
+                //need get attributes?
+                if(record.selected_column_array.empty()) {
                     thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
+                    continue;
                 }
 
-                result_array_.emplace_back(thrift_topk_result);
+                std::string nid = std::to_string(id);
+                std::string attrib_str;
+                IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_);
+
+                AttribMap attrib_map;
+                AttributeSerializer::Decode(attrib_str, attrib_map);
+
+                for(auto& attribute : record.selected_column_array) {
+                    thrift_result.column_map[attribute] = attrib_map[attribute];
+                }
+
+                thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
             }
-            rc.Record("construct result");
-        }
 
-        rc.Record("done");
+            result_array_.emplace_back(thrift_topk_result);
+        }
+        rc.Record("construct result");
 
     } catch (std::exception& ex) {
         error_code_ = SERVER_UNEXPECTED_ERROR;
@@ -366,6 +466,26 @@ ServerError SearchVectorTask::OnExecute() {
     return SERVER_SUCCESS;
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+PingTask::PingTask(const std::string& cmd, std::string& result)
+    : BaseTask(PING_TASK_GROUP),
+      cmd_(cmd),
+      result_(result) {
+
+}
+
+BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
+    return std::shared_ptr<BaseTask>(new PingTask(cmd, result));
+}
+
+ServerError PingTask::OnExecute() {
+    if(cmd_ == "version") {
+        result_ = "v1.2.0";//currently hardcode
+    }
+
+    return SERVER_SUCCESS;
+}
+
 }
 }
 }

cpp/src/server/MegasearchTask.h

@@ -65,6 +65,50 @@ private:
     std::string table_name_;
 };
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class CreateTablePartitionTask : public BaseTask {
+public:
+    static BaseTaskPtr Create(const thrift::CreateTablePartitionParam &param);
+
+protected:
+    CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param);
+
+    ServerError OnExecute() override;
+
+
+private:
+    const thrift::CreateTablePartitionParam &param_;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class DeleteTablePartitionTask : public BaseTask {
+public:
+    static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam &param);
+
+protected:
+    DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param);
+
+    ServerError OnExecute() override;
+
+
+private:
+    const thrift::DeleteTablePartitionParam &param_;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class ShowTablesTask : public BaseTask {
+public:
+    static BaseTaskPtr Create(std::vector<std::string>& tables);
+
+protected:
+    ShowTablesTask(std::vector<std::string>& tables);
+
+    ServerError OnExecute() override;
+
+private:
+    std::vector<std::string>& tables_;
+};
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 class AddVectorTask : public BaseTask {
 public:
@@ -108,6 +152,21 @@ private:
     std::vector<thrift::TopKQueryResult>& result_array_;
 };
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class PingTask : public BaseTask {
+public:
+    static BaseTaskPtr Create(const std::string& cmd, std::string& result);
+
+protected:
+    PingTask(const std::string& cmd, std::string& result);
+
+    ServerError OnExecute() override;
+
+private:
+    std::string cmd_;
+    std::string& result_;
+};
+
 }
 }
 }
\ No newline at end of file

cpp/src/server/RocksIdMapper.cpp

@@ -9,6 +9,7 @@
 #include "utils/Log.h"
 #include "utils/CommonUtil.h"
 
+#include "rocksdb/db.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/options.h"
 
@@ -108,6 +109,19 @@ bool RocksIdMapper::IsGroupExist(const std::string& group) const {
     return IsGroupExistInternal(group);
 }
 
+ServerError RocksIdMapper::AllGroups(std::vector<std::string>& groups) const {
+    groups.clear();
+
+    std::lock_guard<std::mutex> lck(db_mutex_);
+    for(auto& pair : column_handles_) {
+        if(pair.first == ROCKSDB_DEFAULT_GROUP) {
+            continue;
+        }
+        groups.push_back(pair.first);
+    }
+
+    return SERVER_SUCCESS;
+}
 
 ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
     std::lock_guard<std::mutex> lck(db_mutex_);

cpp/src/server/RocksIdMapper.h

@@ -8,13 +8,16 @@
 #include "utils/Error.h"
 #include "VecIdMapper.h"
 
-#include "rocksdb/db.h"
-
 #include <string>
 #include <vector>
 #include <unordered_map>
 #include <mutex>
 
+namespace rocksdb {
+    class DB;
+    class ColumnFamilyHandle;
+}
+
 namespace zilliz {
 namespace vecwise {
 namespace server {
@@ -26,6 +29,7 @@ class RocksIdMapper : public IVecIdMapper{
 
     ServerError AddGroup(const std::string& group) override;
     bool IsGroupExist(const std::string& group) const override;
+    ServerError AllGroups(std::vector<std::string>& groups) const override;
 
     ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
     ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;

cpp/src/server/ServerConfig.h

@@ -8,7 +8,7 @@
 #include "utils/Error.h"
 #include "config/ConfigNode.h"
 
-#include <yaml-cpp/yaml.h>
+#include "yaml-cpp/yaml.h"
 
 namespace zilliz {
 namespace vecwise {

cpp/src/server/VecIdMapper.cpp

@@ -15,6 +15,7 @@
 #include "rocksdb/options.h"
 
 #include <exception>
+#include <unordered_map>
 
 namespace zilliz {
 namespace vecwise {
@@ -30,6 +31,30 @@ IVecIdMapper* IVecIdMapper::GetInstance() {
 #endif
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+class SimpleIdMapper : public IVecIdMapper{
+public:
+    SimpleIdMapper();
+    ~SimpleIdMapper();
+
+    ServerError AddGroup(const std::string& group) override;
+    bool IsGroupExist(const std::string& group) const override;
+    ServerError AllGroups(std::vector<std::string>& groups) const override;
+
+    ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
+    ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
+
+    ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
+    ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
+
+    ServerError Delete(const std::string& nid, const std::string& group = "") override;
+    ServerError DeleteGroup(const std::string& group) override;
+
+private:
+    using ID_MAPPING = std::unordered_map<std::string, std::string>;
+    mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
+};
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 SimpleIdMapper::SimpleIdMapper() {
 
@@ -52,6 +77,15 @@ SimpleIdMapper::IsGroupExist(const std::string& group) const {
     return id_groups_.count(group) > 0;
 }
 
+ServerError SimpleIdMapper::AllGroups(std::vector<std::string>& groups) const {
+    groups.clear();
+
+    for(auto& pair : id_groups_) {
+        groups.push_back(pair.first);
+    }
+
+    return SERVER_SUCCESS;
+}
 
 //not thread-safe
 ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {

cpp/src/server/VecIdMapper.h

@@ -9,11 +9,6 @@
 
 #include <string>
 #include <vector>
-#include <unordered_map>
-
-namespace rocksdb {
-    class DB;
-}
 
 namespace zilliz {
 namespace vecwise {
@@ -27,6 +22,7 @@ public:
 
     virtual ServerError AddGroup(const std::string& group) = 0;
     virtual bool IsGroupExist(const std::string& group) const = 0;
+    virtual ServerError AllGroups(std::vector<std::string>& groups) const = 0;
 
     virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
     virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
@@ -39,28 +35,6 @@ public:
     virtual ServerError DeleteGroup(const std::string& group) = 0;
 };
 
-class SimpleIdMapper : public IVecIdMapper{
-public:
-    SimpleIdMapper();
-    ~SimpleIdMapper();
-
-    ServerError AddGroup(const std::string& group) override;
-    bool IsGroupExist(const std::string& group) const override;
-
-    ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
-    ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
-
-    ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
-    ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
-
-    ServerError Delete(const std::string& nid, const std::string& group = "") override;
-    ServerError DeleteGroup(const std::string& group) override;
-
-private:
-    using ID_MAPPING = std::unordered_map<std::string, std::string>;
-    mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
-};
-
 }
 }
 }

cpp/thirdparty/versions.txt

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Toolchain library versions
+#
+# This file is used by `download_dependencies.sh` and cmake to figure out which
+# version of a dependency to fetch. In order to add a new dependency, add a
+# version variable, e.g. MY_DEP_VERSION and append an entry in the
+# `DEPENDENCIES` array (see the comment on top of the declaration for the
+# format).
+
+BOOST_VERSION=1.70.0
+BZIP2_VERSION=1.0.6
+EASYLOGGINGPP_VERSION=v9.96.7
+FAISS_VERSION=v1.5.2
+GTEST_VERSION=1.8.1
+LAPACK_VERSION=v3.8.0
+LZ4_VERSION=v1.9.1
+OPENBLAS_VERSION=v0.3.6
+PROMETHEUS_VERSION=master
+ROCKSDB_VERSION=v6.0.2
+SNAPPY_VERSION=1.1.7
+SQLITE_VERSION=3280000
+SQLITE_ORM_VERSION=master
+THRIFT_VERSION=0.12.0
+YAMLCPP_VERSION=0.6.2
+ZLIB_VERSION=v1.2.11
+ZSTD_VERSION=v1.4.0
+
+# The first field is the name of the environment variable expected by cmake.
+# This _must_ match what is defined. The second field is the name of the
+# generated archive file. The third field is the url of the project for the
+# given version.
+DEPENDENCIES=(
+  "MEGASEARCH_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz"
+  "MEGASEARCH_BZIP2_URL bzip2-${BZIP2_VERSION}.tar.gz https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz"
+  "MEGASEARCH_EASYLOGGINGPP_URL easyloggingpp-${EASYLOGGINGPP_VERSION}.tar.gz https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz"
+  "MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz"
+  "MEGASEARCH_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+  "MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz
+  "MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz"
+  "MEGASEARCH_OPENBLAS_URL openblas-${OPENBLAS_VERSION}.tar.gz https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz"
+  "MEGASEARCH_PROMETHEUS_URL prometheus-${PROMETHEUS_VERSION}.tar.gz https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz"
+  "MEGASEARCH_ROCKSDB_URL rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz"
+  "MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz"
+  "MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz"
+  "MEGASEARCH_SQLITE_ORM_URL sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz"
+  "MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
+  "MEGASEARCH_YAMLCPP_URL yaml-cpp-${YAMLCPP_VERSION}.tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz"
+  "MEGASEARCH_ZLIB_URL zlib-${ZLIB_VERSION}.tar.gz https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz"
+  "MEGASEARCH_ZSTD_URL zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz"
+ )
+
+# vim: set filetype=sh:
\ No newline at end of file

cpp/unittest/CMakeLists.txt

@@ -5,18 +5,26 @@
 #-------------------------------------------------------------------------------
 link_directories(
         "${CMAKE_BINARY_DIR}/lib"
-        "${VECWISE_THIRD_PARTY_BUILD}/lib"
+        #"${VECWISE_THIRD_PARTY_BUILD}/lib"
+        "${GTEST_PREFIX}/lib/"
 )
+message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib")
 
 set(unittest_srcs
-        ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp ${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc)
+        ${CMAKE_CURRENT_SOURCE_DIR}/vecwise_test.cpp)
+        #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc)
 
 set(unittest_libs
         yaml-cpp
+        gtest
+        gmock
         gtest_main
         gmock_main
+        easyloggingpp
         pthread
-        metrics)
+        metrics
+        openblas
+        gfortran)
 
 add_subdirectory(server)
 add_subdirectory(db)

cpp/unittest/db/CMakeLists.txt

@@ -42,4 +42,4 @@ set(db_libs
         lz4
         )
 
-target_link_libraries(db_test ${unittest_libs} ${db_libs})
+target_link_libraries(db_test ${db_libs} ${unittest_libs})

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -45,7 +45,7 @@ set(wrapper_libs
         zstd
         lz4
         )
-target_link_libraries(wrapper_test ${unittest_libs} ${wrapper_libs})
+target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs})
 
 set(topk_test_src
         topk_test.cpp

cpp/unittest/license/CMakeLists.txt

@@ -16,6 +16,7 @@ link_directories(/usr/local/cuda/lib64)
 link_directories(/usr/lib/x86_64-linux-gnu)
 link_directories(/usr/lib/nvidia-415)
 link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/)
+link_directories(/usr/local/cuda/lib64/stubs/)
 
 set(require_files
         ../../src/license/LicenseLibrary.cpp
@@ -44,4 +45,4 @@ set(db_libs
         boost_serialization
         )
 
-target_link_libraries(license_test ${unittest_libs} ${db_libs})
+target_link_libraries(license_test ${db_libs} ${unittest_libs})

cpp/unittest/metrics/CMakeLists.txt

@@ -57,7 +57,7 @@ set(count_test_src
 add_executable(metrics_test ${count_test_src} ${require_files} )
 
 target_link_libraries(metrics_test
-        ${unittest_libs}
+
         faiss
         cudart
         cublas
@@ -72,5 +72,5 @@ target_link_libraries(metrics_test
 #        prometheus-cpp-core
         pthread
         z
-
+        ${unittest_libs}
         )
\ No newline at end of file

cpp/unittest/server/CMakeLists.txt

@@ -35,23 +35,23 @@ cuda_add_executable(server_test
 
 set(require_libs
         stdc++
-        boost_system
-        boost_filesystem
-        pthread
-        snappy
-        bz2
-        z
-        zstd
         rocksdb
         faiss
         cudart
         cublas
         sqlite3
+        boost_system
+        boost_filesystem
+        snappy
+        z
+        bz2
+        zstd
         lz4
+        pthread
         )
 
 target_link_libraries(server_test
-        ${unittest_libs}
         ${require_libs}
         ${cuda_library}
+        ${unittest_libs}
         )