Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 cf5636e4 编写于 作者: J jinhai
浏览文件
操作

Merge branch 'branch-0.2.0' into 'develop' 

Merge from Branch 0.2.0 to develop branch

See merge request megasearch/vecwise_engine!54

Former-commit-id: b4e9b6239bbf77215f32d40328b3b523bea2e1e9
上级 ee7839fa 47107b66
展开全部 隐藏空白更改
内联 并排
Showing with 14315 addition and 6415 deletion
cpp/CHANGELOG.md
浏览文件 @ cf5636e4
......@@ -10,7 +10,22 @@ Please mark all change in change log and use the ticket from JIRA.
### New Feature
- MS-5 - Implement Auto Archive Feature
- MS-16 - Implement metrics without prometheus
- MS-6 - Implement SDK interface part 1
- MS-21 - Implement SDK interface part 2
- MS-26 - cmake. Add thirdparty packages
- MS-31 - cmake: add prometheus
- MS-33 - cmake: add -j4 to make third party packages build faster
- MS-27 - support gpu config and disable license build config in cmake
- MS-47 - Add query vps metrics
### Task
- MS-1 - Add CHANGELOG.md
- MS-4 - Refactor the vecwise_engine code structure
\ No newline at end of file
- MS-4 - Refactor the vecwise_engine code structure
- MS-20 - Clean Code Part 1
- MS-30 - Use faiss v1.5.2
- MS-32 - Fix thrift error
- MS-34 - Fix prometheus-cpp thirdparty
- MS-37 - Add query, cache usage, disk write speed and file data size metrics
cpp/CMakeLists.txt
浏览文件 @ cf5636e4
......@@ -30,6 +30,7 @@ message(STATUS "MegaSearch version: "
set(MEGASEARCH_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MEGASEARCH_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
......@@ -41,6 +42,7 @@ if (GPU_VERSION STREQUAL "ON")
endif ()
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building vecwise_engine on x86 architecture")
......@@ -54,25 +56,27 @@ else()
set(VECWISE_BUILD_ARCH unknown)
endif()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
# if (GPU_VERSION STREQUAL "ON")
# set(ENABLE_LICENSE "ON")
# add_definitions("-DENABLE_LICENSE")
# endif ()
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif()
if (GPU_VERSION STREQUAL "ON")
set(ENABLE_LICENSE "ON")
add_definitions("-DENABLE_LICENSE")
endif ()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
if (BUILD_UNIT_TEST)
option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON)
endif(BUILD_UNIT_TEST)
#if (BUILD_UNIT_TEST)
# option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON)
#endif(BUILD_UNIT_TEST)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
include_directories(${MEGASEARCH_SOURCE_DIR})
......@@ -82,24 +86,22 @@ link_directories(${MEGASEARCH_BINARY_DIR})
set(VECWISE_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
set(VECWISE_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
#set(VECWISE_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
#set(VECWISE_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build)
add_compile_definitions(PROFILER=${PROFILER})
include_directories(${VECWISE_ENGINE_INCLUDE})
include_directories(${VECWISE_ENGINE_SRC})
include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
#include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
link_directories(${CMAKE_CURRRENT_BINARY_DIR})
link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib)
#link_directories(${VECWISE_THIRD_PARTY_BUILD}/lib64)
#execute_process(COMMAND bash build.sh
# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
add_subdirectory(src)
add_subdirectory(test_client)
if (BUILD_UNIT_TEST)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
......@@ -118,3 +120,5 @@ install(FILES
conf/vecwise_engine_log.conf
DESTINATION
conf)
config_summary()
cpp/build.sh
浏览文件 @ cf5636e4
......@@ -72,7 +72,7 @@ if [[ ${MAKE_CLEAN} = "ON" ]]; then
make clean
fi
make -j || exit 1
make -j 4 || exit 1
if [[ ${BUILD_TYPE} != "Debug" ]]; then
strip src/vecwise_server
......
cpp/cmake/BuildUtils.cmake 0 → 100644
浏览文件 @ cf5636e4
function(ADD_THIRDPARTY_LIB LIB_NAME)
set(options)
set(one_value_args SHARED_LIB STATIC_LIB)
set(multi_value_args DEPS INCLUDE_DIRECTORIES)
cmake_parse_arguments(ARG
"${options}"
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
if(NOT ARG_STATIC_LIB)
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_STATIC_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_SHARED_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
else()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endfunction()
\ No newline at end of file
cpp/cmake/DefineOptions.cmake 0 → 100644
浏览文件 @ cf5636e4
macro(set_option_category name)
set(MEGASEARCH_OPTION_CATEGORY ${name})
list(APPEND "MEGASEARCH_OPTION_CATEGORIES" ${name})
endmacro()
macro(define_option name description default)
option(${name} ${description} ${default})
list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" ${default})
set("${name}_OPTION_TYPE" "bool")
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
macro(define_option_string name description default)
set(${name} ${default} CACHE STRING ${description})
list(APPEND "MEGASEARCH_${MEGASEARCH_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" "\"${default}\"")
set("${name}_OPTION_TYPE" "string")
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endmacro()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT "AUTO")
define_option_string(MEGASEARCH_DEPENDENCY_SOURCE
"Method to use for acquiring MEGASEARCH's build dependencies"
"${MEGASEARCH_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF)
define_option(MEGASEARCH_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
Note that this requires linking Boost statically" ON)
define_option(MEGASEARCH_BOOST_HEADER_ONLY "Use only BOOST headers" OFF)
define_option(MEGASEARCH_WITH_BZ2 "Build with BZ2 compression" ON)
define_option(MEGASEARCH_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON)
define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against"
"-gencode=arch=compute_61,code=sm_61")
define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON)
define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON)
define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON)
define_option(MEGASEARCH_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON)
define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON)
define_option(MEGASEARCH_WITH_SQLITE "Build with SQLite library" ON)
define_option(MEGASEARCH_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
define_option(MEGASEARCH_WITH_THRIFT "Build with Apache Thrift library" ON)
define_option(MEGASEARCH_WITH_YAMLCPP "Build with yaml-cpp library" ON)
define_option(MEGASEARCH_WITH_ZLIB "Build with zlib compression" ON)
if(CMAKE_VERSION VERSION_LESS 3.7)
set(MEGASEARCH_WITH_ZSTD_DEFAULT OFF)
else()
# ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7.
set(MEGASEARCH_WITH_ZSTD_DEFAULT ON)
endif()
define_option(MEGASEARCH_WITH_ZSTD "Build with zstd compression" ${MEGASEARCH_WITH_ZSTD_DEFAULT})
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(MEGASEARCH_USE_STATIC_CRT "Build MEGASEARCH with statically linked CRT" OFF)
endif()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
if (BUILD_UNIT_TEST)
define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON)
else()
define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
message(STATUS "---------------------------------------------------------------------")
message(STATUS "MEGASEARCH version: ${MEGASEARCH_VERSION}")
message(STATUS)
message(STATUS "Build configuration summary:")
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
endif()
foreach(category ${MEGASEARCH_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
set(option_names ${MEGASEARCH_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
set(value "${${name}}")
endif()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
set(description_padding "
")
endif()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach()
endmacro()
cpp/cmake/GoogleTest.cmake 已删除 100644 → 0
浏览文件 @ ee7839fa
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tries to find GTest headers and libraries.
#
# Usage of this module as follows:
#
# find_package(GTest)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# GTest_HOME - When set, this path is inspected instead of standard library
# locations as the root of the GTest installation.
# The environment variable GTEST_HOME overrides this veriable.
#
# This module defines
# GTEST_INCLUDE_DIR, directory containing headers
# GTEST_LIBS, directory containing gtest libraries
# GTEST_STATIC_LIB, path to libgtest.a
# GTEST_SHARED_LIB, path to libgtest's shared library
# GTEST_FOUND, whether gtest has been found
if( NOT "${GTEST_HOME}" STREQUAL "")
file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path )
list( APPEND _gtest_roots ${_native_path} )
elseif ( GTest_HOME )
list( APPEND _gtest_roots ${GTest_HOME} )
endif()
# Try the parameterized roots, if they exist
if ( _gtest_roots )
find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h
PATHS ${_gtest_roots} NO_DEFAULT_PATH
PATH_SUFFIXES "include" )
find_library( GTEST_LIBRARIES NAMES gtest gtest_main
PATHS ${_gtest_roots} NO_DEFAULT_PATH
PATH_SUFFIXES "lib" )
else ()
find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h )
find_library( GTEST_LIBRARIES NAMES gtest )
endif ()
if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
set(GTEST_FOUND TRUE)
get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH )
set(GTEST_LIB_NAME gtest)
set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX})
set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
else ()
set(GTEST_FOUND FALSE)
endif ()
if (GTEST_FOUND)
if (NOT GTest_FIND_QUIETLY)
message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}")
endif ()
else ()
if (NOT GTest_FIND_QUIETLY)
set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ")
if ( _gtest_roots )
set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.")
else ()
set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.")
endif ()
if (GTest_FIND_REQUIRED)
message(FATAL_ERROR "${GTEST_ERR_MSG}")
else (GTest_FIND_REQUIRED)
message(STATUS "${GTEST_ERR_MSG}")
endif (GTest_FIND_REQUIRED)
endif ()
endif ()
mark_as_advanced(
GTEST_INCLUDE_DIR
GTEST_LIBS
GTEST_LIBRARIES
GTEST_STATIC_LIB
GTEST_SHARED_LIB
)
cpp/cmake/ThirdPartyPackages.cmake
浏览文件 @ cf5636e4
此差异已折叠。
cpp/conf/server_config.yaml
浏览文件 @ cf5636e4
server_config:
address: 0.0.0.0
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
transfer_protocol: binary #optional: binary, compact, json
server_mode: thread_pool #optional: simple, thread_pool
gpu_index: 0 #which gpu to be used
......@@ -11,6 +11,15 @@ db_config:
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128
metric_config:
is_startup: true # true is on, false is off
collector: prometheus # prometheus, now we only have prometheus
prometheus_config:
collect_type: pull # pull means prometheus pull the message from megasearch, push means megasearch push metric to push gateway
port: 8080
push_gateway_ip_address: 127.0.0.1
push_gateway_port: 9091
license_config:
license_path: "/tmp/system.license"
......
cpp/conf/server_config_template.yaml
浏览文件 @ cf5636e4
server_config:
address: 0.0.0.0
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
transfer_protocol: binary #optional: binary, compact, json
server_mode: thread_pool #optional: simple, thread_pool
gpu_index: 0 #which gpu to be used
......
cpp/src/CMakeLists.txt
浏览文件 @ cf5636e4
......@@ -4,41 +4,95 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
aux_source_directory(utils utils_files)
aux_source_directory(db db_files)
aux_source_directory(wrapper wrapper_files)
aux_source_directory(metrics metrics_files)
#set(metrics_files
# metrics/Metrics.cpp
# metrics/MetricBase.h
#)
set(license_check_files
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
set(license_generator_src
set(license_generator_files
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
set(service_files
thrift/gen-cpp/VecService.cpp
thrift/gen-cpp/MegasearchService.cpp
thrift/gen-cpp/megasearch_constants.cpp
thrift/gen-cpp/megasearch_types.cpp
)
set(vecwise_engine_src
set(vecwise_engine_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
${cache_files}
${db_files}
${wrapper_files}
# metrics/Metrics.cpp
${metrics_files}
)
set(get_sys_info_src
set(get_sys_info_files
license/GetSysInfo.cpp)
include_directories(/usr/include)
include_directories(/usr/local/cuda/include)
include_directories(thrift/gen-cpp)
#target_link_libraries(megasearch boost_system_static)
#target_link_libraries(megasearch boost_filesystem_static)
#target_link_libraries(megasearch boost_serialization_static)
#target_link_libraries(megasearch bzip2)
#target_link_libraries(megasearch easyloggingpp)
#target_link_libraries(megasearch faiss)
#target_link_libraries(megasearch gtest)
#target_link_libraries(megasearch lapack)
#target_link_libraries(megasearch lz4)
#target_link_libraries(megasearch openblas)
#target_link_libraries(megasearch rocksdb)
#target_link_libraries(megasearch snappy)
#target_link_libraries(megasearch sqlite)
#target_link_libraries(megasearch sqlite_orm)
#target_link_libraries(megasearch thrift)
#target_link_libraries(megasearch yaml-cpp)
#target_link_libraries(megasearch zlib)
#target_link_libraries(megasearch zstd)
set(third_party_libs
easyloggingpp
sqlite
# sqlite_orm
thrift
yaml-cpp
faiss
lapack
openblas
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
civetweb
rocksdb
boost_system_static
boost_filesystem_static
boost_serialization_static
bzip2
lz4
snappy
zlib
zstd
)
if (GPU_VERSION STREQUAL "ON")
link_directories(/usr/local/cuda/lib64)
......@@ -52,6 +106,9 @@ if (GPU_VERSION STREQUAL "ON")
cudart
cublas
libsqlite3.a
libprometheus-cpp-push.a
libprometheus-cpp-pull.a
libprometheus-cpp-core.a
)
else()
set(engine_libs
......@@ -62,6 +119,9 @@ else()
libgfortran.a
libquadmath.a
libsqlite3.a
libprometheus-cpp-push.a
libprometheus-cpp-pull.a
libprometheus-cpp-core.a
)
endif ()
......@@ -80,21 +140,25 @@ if (ENABLE_LICENSE STREQUAL "ON")
endif ()
cuda_add_library(vecwise_engine STATIC ${vecwise_engine_src})
target_link_libraries(vecwise_engine ${engine_libs})
cuda_add_library(vecwise_engine STATIC ${vecwise_engine_files})
target_link_libraries(vecwise_engine ${engine_libs} ${third_party_libs})
add_library(metrics STATIC ${metrics_files})
if (ENABLE_LICENSE STREQUAL "ON")
add_library(vecwise_license STATIC ${license_check_files})
target_link_libraries(vecwise_license ${license_libs})
target_link_libraries(vecwise_license ${license_libs} ${third_party_libs})
endif ()
add_executable(vecwise_server
${config_files}
${server_files}
${utils_files}
${service_files}
${VECWISE_THIRD_PARTY_BUILD}/include/easylogging++.cc
)
#set(metrics_lib
# libprometheus-cpp-push.a
# libprometheus-cpp-pull.a
# libprometheus-cpp-core.a
# )
#add_library(vecwise_engine STATIC ${metrics_files} )
#target_link_libraries(metrics ${metrics_lib})
set(server_libs
vecwise_engine
......@@ -110,20 +174,44 @@ set(server_libs
libzstd.a
liblz4.a
dl
metrics
)
add_executable(vecwise_server
${config_files}
${server_files}
${utils_files}
${service_files}
${metrics_files}
#${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc
)
if (ENABLE_LICENSE STREQUAL "ON")
target_link_libraries(vecwise_server ${server_libs} vecwise_license)
target_link_libraries(vecwise_server ${server_libs} vecwise_license ${third_party_libs})
else ()
target_link_libraries(vecwise_server ${server_libs})
target_link_libraries(vecwise_server ${server_libs} ${third_party_libs})
endif()
if (ENABLE_LICENSE STREQUAL "ON")
add_executable(license_generator ${license_generator_src})
add_executable(get_sys_info ${get_sys_info_src})
target_link_libraries(get_sys_info ${license_libs} vecwise_license)
target_link_libraries(license_generator ${license_libs})
add_executable(get_sys_info ${get_sys_info_files})
add_executable(license_generator ${license_generator_files})
target_link_libraries(get_sys_info ${license_libs} vecwise_license ${third_party_libs})
target_link_libraries(license_generator ${license_libs} ${third_party_libs})
install(TARGETS get_sys_info DESTINATION bin)
install(TARGETS license_generator DESTINATION bin)
endif ()
install(TARGETS vecwise_server DESTINATION bin)
\ No newline at end of file
install(TARGETS vecwise_server DESTINATION bin)
add_subdirectory(sdk)
#target_link_libraries(
# libprometheus-cpp-push.a
# libprometheus-cpp-pull.a
# libprometheus-cpp-core.a
# pthread
# z
# ${CURL_LIBRARIES})
cpp/src/cache/CacheMgr.cpp
浏览文件 @ cf5636e4
......@@ -5,6 +5,7 @@
////////////////////////////////////////////////////////////////////////////////
#include "CacheMgr.h"
#include "metrics/Metrics.h"
namespace zilliz {
namespace vecwise {
......@@ -37,7 +38,7 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) {
if(cache_ == nullptr) {
return nullptr;
}
server::Metrics::GetInstance().CacheAccessTotalIncrement();
return cache_->get(key);
}
......@@ -56,6 +57,7 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) {
}
cache_->insert(key, data);
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index) {
......@@ -65,6 +67,7 @@ void CacheMgr::InsertItem(const std::string& key, const engine::Index_ptr& index
DataObjPtr obj = std::make_shared<DataObj>(index);
cache_->insert(key, obj);
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
void CacheMgr::EraseItem(const std::string& key) {
......@@ -73,6 +76,7 @@ void CacheMgr::EraseItem(const std::string& key) {
}
cache_->erase(key);
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
void CacheMgr::PrintInfo() {
......
cpp/src/db/DB.cpp
浏览文件 @ cf5636e4
......@@ -7,8 +7,6 @@
#include "DBImpl.h"
#include "DBMetaImpl.h"
#include "Env.h"
/* #include "FaissExecutionEngine.h" */
/* #include "Traits.h" */
#include "Factories.h"
namespace zilliz {
......
cpp/src/db/DB.h
浏览文件 @ cf5636e4
......@@ -5,12 +5,13 @@
******************************************************************************/
#pragma once
#include <string>
#include "Options.h"
#include "Meta.h"
#include "Status.h"
#include "Types.h"
#include <string>
namespace zilliz {
namespace vecwise {
namespace engine {
......@@ -21,25 +22,22 @@ class DB {
public:
static void Open(const Options& options, DB** dbptr);
virtual Status add_group(meta::GroupSchema& group_info_) = 0;
virtual Status get_group(meta::GroupSchema& group_info_) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
meta::GroupFilesSchema& group_files_info_) = 0;
virtual Status CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status HasTable(const std::string& table_id_, bool& has_or_not_) = 0;
virtual Status add_vectors(const std::string& group_id_,
virtual Status InsertVectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status Query(const std::string& table_id, size_t k, size_t nq,
const float* vectors, QueryResults& results) = 0;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status Query(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status drop_all() = 0;
virtual Status Size(long& result) = 0;
virtual Status count(const std::string& group_id, long& result) = 0;
virtual Status DropAll() = 0;
DB() = default;
DB(const DB&) = delete;
......
cpp/src/db/DBImpl.h
浏览文件 @ cf5636e4
......@@ -5,16 +5,16 @@
******************************************************************************/
#pragma once
#include <mutex>
#include <condition_variable>
#include <memory>
#include <atomic>
#include "DB.h"
#include "MemManager.h"
#include "Types.h"
#include "FaissExecutionEngine.h"
#include "Traits.h"
#include <mutex>
#include <condition_variable>
#include <memory>
#include <atomic>
namespace zilliz {
namespace vecwise {
namespace engine {
......@@ -28,60 +28,56 @@ namespace meta {
template <typename EngineT>
class DBImpl : public DB {
public:
typedef typename meta::Meta::Ptr MetaPtr;
typedef typename MemManager<EngineT>::Ptr MemManagerPtr;
using MetaPtr = meta::Meta::Ptr;
using MemManagerPtr = typename MemManager<EngineT>::Ptr;
DBImpl(const Options& options);
virtual Status add_group(meta::GroupSchema& group_info) override;
virtual Status get_group(meta::GroupSchema& group_info) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
meta::GroupFilesSchema& group_files_info_) override;
virtual Status CreateTable(meta::TableSchema& table_schema) override;
virtual Status DescribeTable(meta::TableSchema& table_schema) override;
virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
virtual Status add_vectors(const std::string& group_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) override;
virtual Status InsertVectors(const std::string& table_id,
size_t n, const float* vectors, IDNumbers& vector_ids) override;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status Query(const std::string& table_id, size_t k, size_t nq,
const float* vectors, QueryResults& results) override;
virtual Status search(const std::string& group_id, size_t k, size_t nq,
virtual Status Query(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
virtual Status drop_all() override;
virtual Status DropAll() override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status Size(long& result) override;
virtual ~DBImpl();
private:
void background_build_index();
Status build_index(const meta::GroupFileSchema&);
Status try_build_index();
Status merge_files(const std::string& group_id,
void BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema&);
Status TryBuildIndex();
Status MergeFiles(const std::string& table_id,
const meta::DateT& date,
const meta::GroupFilesSchema& files);
Status background_merge_files(const std::string& group_id);
const meta::TableFilesSchema& files);
Status BackgroundMergeFiles(const std::string& table_id);
void try_schedule_compaction();
void start_timer_task(int interval_);
void background_timer_task(int interval_);
void TrySchedule();
void StartTimerTasks(int interval);
void BackgroundTimerTask(int interval);
static void BGWork(void* db);
void background_call();
void background_compaction();
void BackgroundCall();
void BackgroundCompaction();
Env* const _env;
const Options _options;
Env* const env_;
const Options options_;
std::mutex _mutex;
std::condition_variable _bg_work_finish_signal;
bool _bg_compaction_scheduled;
Status _bg_error;
std::atomic<bool> _shutting_down;
std::mutex mutex_;
std::condition_variable bg_work_finish_signal_;
bool bg_compaction_scheduled_;
Status bg_error_;
std::atomic<bool> shutting_down_;
std::mutex build_index_mutex_;
bool bg_build_index_started_;
......@@ -89,8 +85,8 @@ private:
std::thread bg_timer_thread_;
MetaPtr _pMeta;
MemManagerPtr _pMemMgr;
MetaPtr pMeta_;
MemManagerPtr pMemMgr_;
}; // DBImpl
......@@ -99,4 +95,4 @@ private:
} // namespace vecwise
} // namespace zilliz
#include "DBImpl.cpp"
#include "DBImpl.inl"
cpp/src/db/DBMetaImpl.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/DBMetaImpl.h
浏览文件 @ cf5636e4
......@@ -19,55 +19,53 @@ class DBMetaImpl : public Meta {
public:
DBMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info) override;
virtual Status get_group(GroupSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status CreateTable(TableSchema& table_schema) override;
virtual Status DescribeTable(TableSchema& group_info_) override;
virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override;
virtual Status CreateTableFile(TableFileSchema& file_schema) override;
virtual Status DropPartitionsByDates(const std::string& table_id,
const DatesT& dates) override;
virtual Status has_group_file(const std::string& group_id_,
const std::string& file_id_,
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override;
virtual Status GetTableFile(TableFileSchema& file_schema) override;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
GroupFilesSchema& group_files_info_) override;
virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
virtual Status update_files(GroupFilesSchema& files) override;
virtual Status UpdateTableFiles(TableFilesSchema& files) override;
virtual Status files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) override;
virtual Status files_to_search(const std::string& group_id,
virtual Status FilesToSearch(const std::string& table_id,
const DatesT& partition,
DatePartionedGroupFilesSchema& files) override;
DatePartionedTableFilesSchema& files) override;
virtual Status files_to_index(GroupFilesSchema&) override;
virtual Status FilesToMerge(const std::string& table_id,
DatePartionedTableFilesSchema& files) override;
virtual Status cleanup() override;
virtual Status FilesToIndex(TableFilesSchema&) override;
virtual Status cleanup_ttl_files(uint16_t seconds) override;
virtual Status Archive() override;
virtual Status drop_all() override;
virtual Status Size(long& result) override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status CleanUp() override;
virtual ~DBMetaImpl();
virtual Status CleanUpFilesWithTTL(uint16_t seconds) override;
private:
virtual Status DropAll() override;
long GetMicroSecTimeStamp();
Status get_group_no_lock(GroupSchema& group_info);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
void GetGroupFilePath(GroupFileSchema& group_file);
Status initialize();
virtual Status Count(const std::string& table_id, long& result) override;
const DBMetaOptions _options;
virtual ~DBMetaImpl();
private:
Status NextFileId(std::string& file_id);
Status NextTableId(std::string& table_id);
Status DiscardFiles(long to_discard_size);
std::string GetTablePath(const std::string& table_id);
std::string GetTableDatePartitionPath(const std::string& table_id, DateT& date);
void GetTableFilePath(TableFileSchema& group_file);
Status Initialize();
const DBMetaOptions options_;
}; // DBMetaImpl
} // namespace meta
......
cpp/src/db/Env.cpp
浏览文件 @ cf5636e4
......@@ -13,66 +13,66 @@ namespace vecwise {
namespace engine {
Env::Env()
: _bg_work_started(false),
_shutting_down(false) {
: bg_work_started_(false),
shutting_down_(false) {
}
void Env::schedule(void (*function_)(void* arg_), void* arg_) {
std::unique_lock<std::mutex> lock(_bg_work_mutex);
if (_shutting_down) return;
void Env::Schedule(void (*function)(void* arg), void* arg) {
std::unique_lock<std::mutex> lock(bg_work_mutex_);
if (shutting_down_) return;
if (!_bg_work_started) {
_bg_work_started = true;
if (!bg_work_started_) {
bg_work_started_ = true;
std::thread bg_thread(Env::BackgroundThreadEntryPoint, this);
bg_thread.detach();
}
if (_bg_work_queue.empty()) {
_bg_work_cv.notify_one();
if (bg_work_queue_.empty()) {
bg_work_cv_.notify_one();
}
_bg_work_queue.emplace(function_, arg_);
bg_work_queue_.emplace(function, arg);
}
void Env::backgroud_thread_main() {
while (!_shutting_down) {
std::unique_lock<std::mutex> lock(_bg_work_mutex);
while (_bg_work_queue.empty() && !_shutting_down) {
_bg_work_cv.wait(lock);
void Env::BackgroundThreadMain() {
while (!shutting_down_) {
std::unique_lock<std::mutex> lock(bg_work_mutex_);
while (bg_work_queue_.empty() && !shutting_down_) {
bg_work_cv_.wait(lock);
}
if (_shutting_down) break;
if (shutting_down_) break;
assert(!_bg_work_queue.empty());
auto bg_function = _bg_work_queue.front()._function;
void* bg_arg = _bg_work_queue.front()._arg;
_bg_work_queue.pop();
assert(!bg_work_queue_.empty());
auto bg_function = bg_work_queue_.front().function_;
void* bg_arg = bg_work_queue_.front().arg_;
bg_work_queue_.pop();
lock.unlock();
bg_function(bg_arg);
}
std::unique_lock<std::mutex> lock(_bg_work_mutex);
_bg_work_started = false;
_bg_work_cv.notify_all();
std::unique_lock<std::mutex> lock(bg_work_mutex_);
bg_work_started_ = false;
bg_work_cv_.notify_all();
}
void Env::Stop() {
{
std::unique_lock<std::mutex> lock(_bg_work_mutex);
if (_shutting_down || !_bg_work_started) return;
std::unique_lock<std::mutex> lock(bg_work_mutex_);
if (shutting_down_ || !bg_work_started_) return;
}
_shutting_down = true;
shutting_down_ = true;
{
std::unique_lock<std::mutex> lock(_bg_work_mutex);
if (_bg_work_queue.empty()) {
_bg_work_cv.notify_one();
std::unique_lock<std::mutex> lock(bg_work_mutex_);
if (bg_work_queue_.empty()) {
bg_work_cv_.notify_one();
}
while (_bg_work_started) {
_bg_work_cv.wait(lock);
while (bg_work_started_) {
bg_work_cv_.wait(lock);
}
}
_shutting_down = false;
shutting_down_ = false;
}
Env::~Env() {}
......
cpp/src/db/Env.h
浏览文件 @ cf5636e4
......@@ -22,7 +22,7 @@ public:
Env(const Env&) = delete;
Env& operator=(const Env&) = delete;
void schedule(void (*function_)(void* arg_), void* arg_);
void Schedule(void (*function)(void* arg), void* arg);
virtual void Stop();
......@@ -31,25 +31,24 @@ public:
static Env* Default();
protected:
void backgroud_thread_main();
void BackgroundThreadMain();
static void BackgroundThreadEntryPoint(Env* env) {
env->backgroud_thread_main();
env->BackgroundThreadMain();
}
struct BGWork {
explicit BGWork(void (*function_)(void*), void* arg_)
: _function(function_), _arg(arg_) {}
explicit BGWork(void (*function)(void*), void* arg)
: function_(function), arg_(arg) {}
void (* const _function)(void*);
void* const _arg;
void (* const function_)(void*);
void* const arg_;
};
std::mutex _bg_work_mutex;
std::condition_variable _bg_work_cv;
std::queue<BGWork> _bg_work_queue;
bool _bg_work_started;
std::atomic<bool> _shutting_down;
std::mutex bg_work_mutex_;
std::condition_variable bg_work_cv_;
std::queue<BGWork> bg_work_queue_;
bool bg_work_started_;
std::atomic<bool> shutting_down_;
}; // Env
} // namespace engine
......
cpp/src/db/Exception.h 0 → 100644
浏览文件 @ cf5636e4
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <exception>
#include <string>
namespace zilliz {
namespace vecwise {
namespace engine {
class Exception : public std::exception {
public:
Exception(const std::string& message)
: message_(message) {
}
Exception()
: message_() {
}
virtual const char* what() const throw() {
if (message_.empty()) {
return "Default Exception.";
} else {
return message_.c_str();
}
}
virtual ~Exception() throw() {};
protected:
std::string message_;
};
class InvalidArgumentException : public Exception {
public:
InvalidArgumentException() : Exception("Invalid Argument"){};
InvalidArgumentException(const std::string& message) : Exception(message) {};
};
class OutOfRangeException : public Exception {
public:
OutOfRangeException() : Exception("Out Of Range"){};
OutOfRangeException(const std::string& message) : Exception(message) {};
};
} // namespace engine
} // namespace vecwise
} // namespace zilliz
cpp/src/db/ExecutionEngine.cpp
浏览文件 @ cf5636e4
......@@ -3,9 +3,10 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include <easylogging++.h>
#include "ExecutionEngine.h"
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace engine {
......
cpp/src/db/ExecutionEngine.h
浏览文件 @ cf5636e4
......@@ -5,11 +5,11 @@
******************************************************************************/
#pragma once
#include "Status.h"
#include <vector>
#include <memory>
#include "Status.h"
namespace zilliz {
namespace vecwise {
namespace engine {
......
cpp/src/db/Factories.cpp
浏览文件 @ cf5636e4
......@@ -3,6 +3,11 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "Factories.h"
#include "DBImpl.h"
#include "FaissExecutionEngine.h"
#include "Traits.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
......@@ -11,12 +16,6 @@
#include <assert.h>
#include <easylogging++.h>
#include "Factories.h"
#include "DBImpl.h"
#include "FaissExecutionEngine.h"
#include "Traits.h"
namespace zilliz {
namespace vecwise {
namespace engine {
......
cpp/src/db/Factories.h
浏览文件 @ cf5636e4
......@@ -3,15 +3,15 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <string>
#include <memory>
#include "DB.h"
#include "DBMetaImpl.h"
#include "Options.h"
#include <string>
#include <memory>
namespace zilliz {
namespace vecwise {
namespace engine {
......
cpp/src/db/FaissExecutionEngine.h
浏览文件 @ cf5636e4
......@@ -5,11 +5,11 @@
******************************************************************************/
#pragma once
#include "ExecutionEngine.h"
#include <memory>
#include <string>
#include "ExecutionEngine.h"
namespace faiss {
class Index;
}
......@@ -22,7 +22,7 @@ namespace engine {
template<class IndexTrait>
class FaissExecutionEngine : public ExecutionEngine<FaissExecutionEngine<IndexTrait>> {
public:
typedef std::shared_ptr<FaissExecutionEngine<IndexTrait>> Ptr;
using Ptr = std::shared_ptr<FaissExecutionEngine<IndexTrait>>;
FaissExecutionEngine(uint16_t dimension, const std::string& location);
FaissExecutionEngine(std::shared_ptr<faiss::Index> index, const std::string& location);
......@@ -53,7 +53,9 @@ public:
Ptr BuildIndex(const std::string&);
Status Cache();
protected:
std::shared_ptr<faiss::Index> pIndex_;
std::string location_;
};
......@@ -63,4 +65,4 @@ protected:
} // namespace vecwise
} // namespace zilliz
#include "FaissExecutionEngine.cpp"
#include "FaissExecutionEngine.inl"
cpp/src/db/FaissExecutionEngine.cpp → cpp/src/db/FaissExecutionEngine.inl
浏览文件 @ cf5636e4
......@@ -3,8 +3,9 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#ifndef FAISSEXECUTIONENGINE_CPP__
#define FAISSEXECUTIONENGINE_CPP__
#pragma once
#include "FaissExecutionEngine.h"
#include <easylogging++.h>
#include <faiss/AutoTune.h>
......@@ -14,8 +15,8 @@
#include <wrapper/Index.h>
#include <wrapper/IndexBuilder.h>
#include <cache/CpuCacheMgr.h>
#include "metrics/Metrics.h"
#include "FaissExecutionEngine.h"
namespace zilliz {
namespace vecwise {
......@@ -47,12 +48,12 @@ size_t FaissExecutionEngine<IndexTrait>::Count() const {
template<class IndexTrait>
size_t FaissExecutionEngine<IndexTrait>::Size() const {
return (size_t)(Count() * pIndex_->d);
return (size_t)(Count() * pIndex_->d)*sizeof(float);
}
template<class IndexTrait>
size_t FaissExecutionEngine<IndexTrait>::PhysicalSize() const {
return (size_t)(Size()*sizeof(float));
return (size_t)(Count() * pIndex_->d)*sizeof(float);
}
template<class IndexTrait>
......@@ -65,6 +66,7 @@ template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Load() {
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool to_cache = false;
auto start_time = METRICS_NOW_TIME;
if (!index) {
index = read_index(location_);
to_cache = true;
......@@ -74,6 +76,16 @@ Status FaissExecutionEngine<IndexTrait>::Load() {
pIndex_ = index->data();
if (to_cache) {
Cache();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().FaissDiskLoadDurationSecondsHistogramObserve(total_time);
double total_size = (pIndex_->d) * (pIndex_->ntotal) * 4;
server::Metrics::GetInstance().FaissDiskLoadSizeBytesHistogramObserve(total_size);
// server::Metrics::GetInstance().FaissDiskLoadIOSpeedHistogramObserve(total_size/double(total_time));
server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size/double(total_time));
}
return Status::OK();
}
......@@ -135,5 +147,3 @@ Status FaissExecutionEngine<IndexTrait>::Cache() {
} // namespace engine
} // namespace vecwise
} // namespace zilliz
#endif
cpp/src/db/IDGenerator.cpp
浏览文件 @ cf5636e4
......@@ -3,30 +3,29 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "IDGenerator.h"
#include <chrono>
#include <assert.h>
#include <iostream>
#include "IDGenerator.h"
namespace zilliz {
namespace vecwise {
namespace engine {
IDGenerator::~IDGenerator() {}
IDNumber SimpleIDGenerator::getNextIDNumber() {
IDNumber SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO;
}
void SimpleIDGenerator::nextIDNumbers(size_t n, IDNumbers& ids) {
void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
if (n > MAX_IDS_PER_MICRO) {
nextIDNumbers(n-MAX_IDS_PER_MICRO, ids);
nextIDNumbers(MAX_IDS_PER_MICRO, ids);
NextIDNumbers(n-MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids);
return;
}
if (n <= 0) {
......@@ -41,12 +40,11 @@ void SimpleIDGenerator::nextIDNumbers(size_t n, IDNumbers& ids) {
for (int pos=0; pos<n; ++pos) {
ids.push_back(micros+pos);
}
}
void SimpleIDGenerator::getNextIDNumbers(size_t n, IDNumbers& ids) {
void SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
ids.clear();
nextIDNumbers(n, ids);
NextIDNumbers(n, ids);
}
......
cpp/src/db/IDGenerator.h
浏览文件 @ cf5636e4
......@@ -5,17 +5,19 @@
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include <vector>
#include "Types.h"
#include <cstddef>
#include <vector>
namespace zilliz {
namespace vecwise {
namespace engine {
class IDGenerator {
public:
virtual IDNumber getNextIDNumber() = 0;
virtual void getNextIDNumbers(size_t n, IDNumbers& ids) = 0;
virtual IDNumber GetNextIDNumber() = 0;
virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) = 0;
virtual ~IDGenerator();
......@@ -24,11 +26,11 @@ public:
class SimpleIDGenerator : public IDGenerator {
public:
virtual IDNumber getNextIDNumber() override;
virtual void getNextIDNumbers(size_t n, IDNumbers& ids) override;
virtual IDNumber GetNextIDNumber() override;
virtual void GetNextIDNumbers(size_t n, IDNumbers& ids) override;
private:
void nextIDNumbers(size_t n, IDNumbers& ids);
void NextIDNumbers(size_t n, IDNumbers& ids);
const size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator
......
cpp/src/db/LocalMetaImpl.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <sys/stat.h>
#include <unistd.h>
#include <sstream>
#include <iostream>
#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <fstream>
#include "LocalMetaImpl.h"
#include "IDGenerator.h"
namespace zilliz {
namespace vecwise {
namespace engine {
namespace meta {
long LocalMetaImpl::GetFileSize(const std::string& filename)
{
struct stat stat_buf;
int rc = stat(filename.c_str(), &stat_buf);
return rc == 0 ? stat_buf.st_size : -1;
}
std::string LocalMetaImpl::GetGroupPath(const std::string& group_id) {
return _options.path + "/" + group_id;
}
std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, DateT& date) {
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date;
return ss.str();
}
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == GroupFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g;
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
return ss.str();
}
std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_path) {
return group_path + "/" + "meta";
}
std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(group_id));
}
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info) {
boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data();
auto dimension = ptree.get_child("dimension").data();
/* std::cout << dimension << std::endl; */
/* std::cout << files_cnt << std::endl; */
group_info.id = std::stoi(group_info.group_id);
group_info.files_cnt = std::stoi(files_cnt);
group_info.dimension = std::stoi(dimension);
group_info.location = GetGroupPath(group_info.group_id);
return Status::OK();
}
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info) {
group_info.group_id = group_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info);
}
LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_)
: _options(options_) {
initialize();
}
Status LocalMetaImpl::initialize() {
if (boost::filesystem::is_directory(_options.path)) {
}
else if (!boost::filesystem::create_directory(_options.path)) {
return Status::InvalidDBPath("Cannot Create " + _options.path);
}
return Status::OK();
}
Status LocalMetaImpl::add_group(GroupSchema& group_info) {
std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.group_id == "") {
std::stringstream ss;
ss << id;
real_gid = ss.str();
} else {
real_gid = group_info.group_id;
}
bool group_exist;
has_group(real_gid, group_exist);
if (group_exist) {
return Status::GroupError("Group Already Existed " + real_gid);
}
if (!boost::filesystem::create_directory(GetGroupPath(real_gid))) {
return Status::GroupError("Cannot Create Group " + real_gid);
}
group_info.group_id = real_gid;
group_info.files_cnt = 0;
group_info.id = 0;
group_info.location = GetGroupPath(real_gid);
boost::property_tree::ptree out;
out.put("files_cnt", group_info.files_cnt);
out.put("dimension", group_info.dimension);
boost::property_tree::write_json(GetGroupMetaPath(real_gid), out);
return Status::OK();
}
Status LocalMetaImpl::get_group(GroupSchema& group_info) {
bool group_exist;
has_group(group_info.group_id, group_exist);
if (!group_exist) {
return Status::NotFound("Group " + group_info.group_id + " Not Found");
}
return GetGroupMetaInfo(group_info.group_id, group_info);
}
Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
has_or_not = boost::filesystem::is_directory(GetGroupPath(group_id));
return Status::OK();
}
Status LocalMetaImpl::add_group_file(GroupFileSchema& group_file_info) {
GroupSchema group_info;
/* auto status = get_group(group_info); */
/* if (!status.ok()) { */
/* return status; */
/* } */
/* auto location = GetNextGroupFileLocationByPartition(group_id, date, file_type); */
/* group_file_info.group_id = group_id; */
/* group_file_info.dimension = group_info.dimension; */
/* group_file_info.location = location; */
/* group_file_info.date = date; */
return Status::OK();
}
Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
files.clear();
std::string suffix;
boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string();
GroupSchema group_info;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string();
for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) {
auto location = fItr->path().string();
suffix = location.substr(location.find_last_of('.') + 1);
if (suffix == "index") continue;
if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
std::cout << "[About to index] " << location << std::endl;
GroupFileSchema f;
f.location = location;
/* f.group_id = group_id; */
f.dimension = group_info.dimension;
files.push_back(f);
}
}
}
return Status::OK();
}
Status LocalMetaImpl::files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) {
files.clear();
/* std::string suffix; */
/* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */
/* GroupSchema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */
/* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */
/* auto location = fItr->path().string(); */
/* suffix = location.substr(location.find_last_of('.') + 1); */
/* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */
/* GroupFileSchema f; */
/* f.location = location; */
/* f.group_id = group_id; */
/* f.dimension = group_info.dimension; */
/* files.push_back(f); */
/* } */
/* } */
/* } */
return Status::OK();
}
Status LocalMetaImpl::has_group_file(const std::string& group_id_,
const std::string& file_id_,
bool& has_or_not_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_files(const std::string& group_id_,
const int date_delta_,
GroupFilesSchema& group_files_info_) {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::update_group_file(GroupFileSchema& group_file_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::update_files(GroupFilesSchema& files) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::cleanup() {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::drop_all() {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::count(const std::string& group_id, long& result) {
// PXU TODO
return Status::OK();
}
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace zilliz
cpp/src/db/LocalMetaImpl.h 已删除 100644 → 0
浏览文件 @ ee7839fa
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "Meta.h"
#include "Options.h"
namespace zilliz {
namespace vecwise {
namespace engine {
namespace meta {
class LocalMetaImpl : public Meta {
public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info_) override;
virtual Status get_group(GroupSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0;
virtual Status has_group_file(const std::string& group_id_,
const std::string& file_id_,
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
GroupFilesSchema& group_files_info_) override;
virtual Status update_files(GroupFilesSchema& files) override;
virtual Status cleanup() override;
virtual Status files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) override;
virtual Status files_to_index(GroupFilesSchema&) override;
virtual Status cleanup_ttl_files(uint16_t seconds) override;
virtual Status count(const std::string& group_id, long& result) override;
virtual Status drop_all() override;
private:
Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupMetaPath(const std::string& group_id);
Status CreateGroupMeta(const GroupSchema& group_schema);
long GetFileSize(const std::string& filename);
Status initialize();
const DBMetaOptions _options;
}; // LocalMetaImpl
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace zilliz
cpp/src/db/MemManager.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/Meta.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/Meta.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/test_client/src/Log.h → cpp/src/db/MetaConsts.h
浏览文件 @ cf5636e4
......@@ -5,22 +5,28 @@
******************************************************************************/
#pragma once
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace client {
namespace engine {
namespace meta {
const size_t K = 1024UL;
const size_t M = K*K;
const size_t G = K*M;
const size_t T = K*G;
#define CLIENT_DOMAIN_NAME "[CLIENT] "
#define CLIENT_ERROR_TEXT "CLIENT Error:"
const size_t S_PS = 1UL;
const size_t MS_PS = 1000*S_PS;
const size_t US_PS = 1000*MS_PS;
const size_t NS_PS = 1000*US_PS;
#define CLIENT_LOG_TRACE LOG(TRACE) << CLIENT_DOMAIN_NAME
#define CLIENT_LOG_DEBUG LOG(DEBUG) << CLIENT_DOMAIN_NAME
#define CLIENT_LOG_INFO LOG(INFO) << CLIENT_DOMAIN_NAME
#define CLIENT_LOG_WARNING LOG(WARNING) << CLIENT_DOMAIN_NAME
#define CLIENT_LOG_ERROR LOG(ERROR) << CLIENT_DOMAIN_NAME
#define CLIENT_LOG_FATAL LOG(FATAL) << CLIENT_DOMAIN_NAME
const size_t SECOND = 1UL;
const size_t M_SEC = 60*SECOND;
const size_t H_SEC = 60*M_SEC;
const size_t D_SEC = 24*H_SEC;
const size_t W_SEC = 7*D_SEC;
} // namespace sql
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace zilliz
} // namespace server
cpp/src/db/MetaTypes.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/Options.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/Options.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/db/db_connection.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/db/db_connection.h 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/license/LicenseCheck.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/license/LicenseCheck.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/main.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/metrics/MetricBase.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/metrics/Metrics.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/metrics/Metrics.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/metrics/PrometheusMetrics.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/metrics/PrometheusMetrics.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/CMakeLists.txt 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/MegaSearch.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/sdk/examples/CMakeLists.txt 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/examples/simple/CMakeLists.txt 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/examples/simple/src/ClientTest.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/client/ClientProxy.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/client/ClientProxy.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/client/ThriftClient.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/interface/ConnectionImpl.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/interface/ConnectionImpl.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/util/ConvertUtil.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/util/ConvertUtil.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/sdk/src/util/Exception.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/MegasearchHandler.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/MegasearchHandler.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/MegasearchTask.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/RocksIdMapper.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/RocksIdMapper.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/Server.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/ServerConfig.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/VecIdMapper.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/VecIdMapper.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/server/VecServiceHandler.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/server/VecServiceHandler.h 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/server/VecServiceTask.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/thrift/cpp_gen.sh
浏览文件 @ cf5636e4
#!/bin/bash
../../third_party/build/bin/thrift -r --gen cpp ./megasearch.thrift
thrift -r --gen cpp ./megasearch.thrift
cpp/src/thrift/gen-cpp/MegasearchService.h 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/thrift/gen-cpp/VecService.h 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/thrift/gen-py/py_sample.py 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/thrift/megasearch.thrift
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/thrift/py_gen.sh 已删除 100755 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/src/wrapper/IndexBuilder.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/src/wrapper/IndexBuilder.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/test_client/CMakeLists.txt 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/test_client/src/ClientTest.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/test_client/src/FaissTest.cpp 已删除 100644 → 0
浏览文件 @ ee7839fa
此差异已折叠。
cpp/thirdparty/versions.txt 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/CMakeLists.txt
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/db/CMakeLists.txt
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/db/db_tests.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/db/meta_tests.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/db/utils.cpp
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/db/utils.h
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/license/CMakeLists.txt
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/metrics/CMakeLists.txt 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/metrics/metrics_test.cpp 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
cpp/unittest/server/CMakeLists.txt
浏览文件 @ cf5636e4
此差异已折叠。
python/CHANGELOG.md 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
python/sdk/.gitignore 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
python/sdk/ClientInterface.py 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
python/sdk/Status.py 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
python/sdk/exceptions.py 0 → 100644
浏览文件 @ cf5636e4
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或