Skip to content

M
milvus

milvus / milvus
12 个月 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 bede7775 编写于 作者: Y youny626
浏览文件
操作

clang format

上级 29e8ef11
展开全部 隐藏空白更改
内联 并排
Showing with 12900 addition and 13942 deletion
core/src/cache/Cache.inl
浏览文件 @ bede7775
......@@ -15,24 +15,18 @@
// specific language governing permissions and limitations
// under the License.
namespace milvus {
namespace cache {
constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85;
template<typename ItemObj>
template <typename ItemObj>
Cache<ItemObj>::Cache(int64_t capacity, uint64_t cache_max_count)
: usage_(0),
capacity_(capacity),
freemem_percent_(DEFAULT_THRESHHOLD_PERCENT),
lru_(cache_max_count) {
// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
: usage_(0), capacity_(capacity), freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), lru_(cache_max_count) {
// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity)
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::set_capacity(int64_t capacity) {
if (capacity > 0) {
......@@ -41,23 +35,23 @@ Cache<ItemObj>::set_capacity(int64_t capacity) {
}
}
template<typename ItemObj>
template <typename ItemObj>
size_t
Cache<ItemObj>::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.size();
}
template<typename ItemObj>
template <typename ItemObj>
bool
Cache<ItemObj>::exists(const std::string &key) {
Cache<ItemObj>::exists(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.exists(key);
}
template<typename ItemObj>
template <typename ItemObj>
ItemObj
Cache<ItemObj>::get(const std::string &key) {
Cache<ItemObj>::get(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
if (!lru_.exists(key)) {
return nullptr;
......@@ -66,60 +60,59 @@ Cache<ItemObj>::get(const std::string &key) {
return lru_.get(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::insert(const std::string &key, const ItemObj &item) {
Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
if (item == nullptr) {
return;
}
// if(item->size() > capacity_) {
// SERVER_LOG_ERROR << "Item size " << item->size()
// << " is too large to insert into cache, capacity " << capacity_;
// return;
// }
// if(item->size() > capacity_) {
// SERVER_LOG_ERROR << "Item size " << item->size()
// << " is too large to insert into cache, capacity " << capacity_;
// return;
// }
//calculate usage
// calculate usage
{
std::lock_guard<std::mutex> lock(mutex_);
//if key already exist, subtract old item size
// if key already exist, subtract old item size
if (lru_.exists(key)) {
const ItemObj &old_item = lru_.get(key);
const ItemObj& old_item = lru_.get(key);
usage_ -= old_item->Size();
}
//plus new item size
// plus new item size
usage_ += item->Size();
}
//if usage exceed capacity, free some items
// if usage exceed capacity, free some items
if (usage_ > capacity_) {
SERVER_LOG_DEBUG << "Current usage " << usage_
<< " exceeds cache capacity " << capacity_
SERVER_LOG_DEBUG << "Current usage " << usage_ << " exceeds cache capacity " << capacity_
<< ", start free memory";
free_memory();
}
//insert new item
// insert new item
{
std::lock_guard<std::mutex> lock(mutex_);
lru_.put(key, item);
SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size()
<< " bytes into cache, usage: " << usage_ << " bytes";
SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size() << " bytes into cache, usage: " << usage_
<< " bytes";
}
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::erase(const std::string &key) {
Cache<ItemObj>::erase(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
if (!lru_.exists(key)) {
return;
}
const ItemObj &old_item = lru_.get(key);
const ItemObj& old_item = lru_.get(key);
usage_ -= old_item->Size();
SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size();
......@@ -127,7 +120,7 @@ Cache<ItemObj>::erase(const std::string &key) {
lru_.erase(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::clear() {
std::lock_guard<std::mutex> lock(mutex_);
......@@ -137,15 +130,16 @@ Cache<ItemObj>::clear() {
}
/* free memory space when CACHE occupation exceed its capacity */
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::free_memory() {
if (usage_ <= capacity_) return;
if (usage_ <= capacity_)
return;
int64_t threshhold = capacity_ * freemem_percent_;
int64_t delta_size = usage_ - threshhold;
if (delta_size <= 0) {
delta_size = 1;//ensure at least one item erased
delta_size = 1; // ensure at least one item erased
}
std::set<std::string> key_array;
......@@ -156,8 +150,8 @@ Cache<ItemObj>::free_memory() {
auto it = lru_.rbegin();
while (it != lru_.rend() && released_size < delta_size) {
auto &key = it->first;
auto &obj_ptr = it->second;
auto& key = it->first;
auto& obj_ptr = it->second;
key_array.emplace(key);
released_size += obj_ptr->Size();
......@@ -167,14 +161,14 @@ Cache<ItemObj>::free_memory() {
SERVER_LOG_DEBUG << "to be released memory size: " << released_size;
for (auto &key : key_array) {
for (auto& key : key_array) {
erase(key);
}
print();
}
template<typename ItemObj>
template <typename ItemObj>
void
Cache<ItemObj>::print() {
size_t cache_count = 0;
......@@ -188,7 +182,5 @@ Cache<ItemObj>::print() {
SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes";
}
} // namespace cache
} // namespace milvus
} // namespace cache
} // namespace milvus
core/src/cache/CacheMgr.inl
浏览文件 @ bede7775
......@@ -15,21 +15,18 @@
// specific language governing permissions and limitations
// under the License.
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
CacheMgr<ItemObj>::CacheMgr() {
}
template<typename ItemObj>
template <typename ItemObj>
CacheMgr<ItemObj>::~CacheMgr() {
}
template<typename ItemObj>
template <typename ItemObj>
uint64_t
CacheMgr<ItemObj>::ItemCount() const {
if (cache_ == nullptr) {
......@@ -37,12 +34,12 @@ CacheMgr<ItemObj>::ItemCount() const {
return 0;
}
return (uint64_t) (cache_->size());
return (uint64_t)(cache_->size());
}
template<typename ItemObj>
template <typename ItemObj>
bool
CacheMgr<ItemObj>::ItemExists(const std::string &key) {
CacheMgr<ItemObj>::ItemExists(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return false;
......@@ -51,9 +48,9 @@ CacheMgr<ItemObj>::ItemExists(const std::string &key) {
return cache_->exists(key);
}
template<typename ItemObj>
template <typename ItemObj>
ItemObj
CacheMgr<ItemObj>::GetItem(const std::string &key) {
CacheMgr<ItemObj>::GetItem(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return nullptr;
......@@ -62,9 +59,9 @@ CacheMgr<ItemObj>::GetItem(const std::string &key) {
return cache_->get(key);
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
......@@ -74,9 +71,9 @@ CacheMgr<ItemObj>::InsertItem(const std::string &key, const ItemObj &data) {
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::EraseItem(const std::string &key) {
CacheMgr<ItemObj>::EraseItem(const std::string& key) {
if (cache_ == nullptr) {
SERVER_LOG_ERROR << "Cache doesn't exist";
return;
......@@ -86,7 +83,7 @@ CacheMgr<ItemObj>::EraseItem(const std::string &key) {
server::Metrics::GetInstance().CacheAccessTotalIncrement();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::PrintInfo() {
if (cache_ == nullptr) {
......@@ -97,7 +94,7 @@ CacheMgr<ItemObj>::PrintInfo() {
cache_->print();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::ClearCache() {
if (cache_ == nullptr) {
......@@ -108,7 +105,7 @@ CacheMgr<ItemObj>::ClearCache() {
cache_->clear();
}
template<typename ItemObj>
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheUsage() const {
if (cache_ == nullptr) {
......@@ -119,7 +116,7 @@ CacheMgr<ItemObj>::CacheUsage() const {
return cache_->usage();
}
template<typename ItemObj>
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheCapacity() const {
if (cache_ == nullptr) {
......@@ -130,7 +127,7 @@ CacheMgr<ItemObj>::CacheCapacity() const {
return cache_->capacity();
}
template<typename ItemObj>
template <typename ItemObj>
void
CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
if (cache_ == nullptr) {
......@@ -140,6 +137,5 @@ CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
cache_->set_capacity(capacity);
}
} // namespace cache
} // namespace milvus
} // namespace cache
} // namespace milvus
core/src/config.h.in
浏览文件 @ bede7775
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#cmakedefine MILVUS_VERSION "@MILVUS_VERSION@"
#cmakedefine BUILD_TYPE "@BUILD_TYPE@"
#cmakedefine BUILD_TIME @BUILD_TIME@
\ No newline at end of file
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ bede7775
此差异已折叠。
core/src/external/nlohmann/json.hpp
浏览文件 @ bede7775
此差异已折叠。
core/src/index/CMakeLists.txt
浏览文件 @ bede7775
......@@ -30,38 +30,38 @@ set(CMAKE_CXX_STANDARD 14)
set(KNOWHERE_VERSION_MAJOR "${knowhere_VERSION_MAJOR}")
set(KNOWHERE_VERSION_MINOR "${knowhere_VERSION_MINOR}")
set(KNOWHERE_VERSION_PATCH "${knowhere_VERSION_PATCH}")
if(KNOWHERE_VERSION_MAJOR STREQUAL ""
if (KNOWHERE_VERSION_MAJOR STREQUAL ""
OR KNOWHERE_VERSION_MINOR STREQUAL ""
OR KNOWHERE_VERSION_PATCH STREQUAL "")
message(FATAL_ERROR "Failed to determine Knowhere version from '${KNOWHERE_VERSION}'")
endif()
endif ()
message(STATUS "Knowhere version: "
"${KNOWHERE_VERSION_MAJOR}.${KNOWHERE_VERSION_MINOR}.${KNOWHERE_VERSION_PATCH} "
"(full: '${KNOWHERE_VERSION}')")
# if no build build type is specified, default to release builds
if(NOT CMAKE_BUILD_TYPE)
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif(NOT CMAKE_BUILD_TYPE)
endif (NOT CMAKE_BUILD_TYPE)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message(STATUS "building milvus_engine on x86 architecture")
set(KNOWHERE_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message(STATUS "building milvus_engine on ppc architecture")
set(KNOWHERE_BUILD_ARCH ppc64le)
else()
else ()
message(WARNING "unknown processor type")
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(KNOWHERE_BUILD_ARCH unknown)
endif()
endif ()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
else()
else ()
set(BUILD_TYPE "debug")
endif()
endif ()
message(STATUS "Build type = ${BUILD_TYPE}")
set(INDEX_SOURCE_DIR ${PROJECT_SOURCE_DIR})
......@@ -103,12 +103,12 @@ add_subdirectory(knowhere)
if (BUILD_COVERAGE STREQUAL "ON")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
endif()
endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
if(KNOWHERE_BUILD_TESTS)
if (KNOWHERE_BUILD_TESTS)
add_subdirectory(unittest)
endif()
endif ()
config_summary()
core/src/index/cmake/BuildUtilsCore.cmake
浏览文件 @ bede7775
# Define a function that check last file modification
function(Check_Last_Modify cache_check_lists_file_path working_dir last_modified_commit_id)
if(EXISTS "${working_dir}")
if(EXISTS "${cache_check_lists_file_path}")
if (EXISTS "${working_dir}")
if (EXISTS "${cache_check_lists_file_path}")
set(GIT_LOG_SKIP_NUM 0)
set(_MATCH_ALL ON CACHE BOOL "Match all")
set(_LOOP_STATUS ON CACHE BOOL "Whether out of loop")
file(STRINGS ${cache_check_lists_file_path} CACHE_IGNORE_TXT)
while(_LOOP_STATUS)
foreach(_IGNORE_ENTRY ${CACHE_IGNORE_TXT})
if(NOT _IGNORE_ENTRY MATCHES "^[^#]+")
while (_LOOP_STATUS)
foreach (_IGNORE_ENTRY ${CACHE_IGNORE_TXT})
if (NOT _IGNORE_ENTRY MATCHES "^[^#]+")
continue()
endif()
endif ()
set(_MATCH_ALL OFF)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --name-status --pretty= WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE CHANGE_FILES)
if(NOT CHANGE_FILES STREQUAL "")
if (NOT CHANGE_FILES STREQUAL "")
string(REPLACE "\n" ";" _CHANGE_FILES ${CHANGE_FILES})
foreach(_FILE_ENTRY ${_CHANGE_FILES})
foreach (_FILE_ENTRY ${_CHANGE_FILES})
string(REGEX MATCH "[^ \t]+$" _FILE_NAME ${_FILE_ENTRY})
execute_process(COMMAND sh -c "echo ${_FILE_NAME} | grep ${_IGNORE_ENTRY}" RESULT_VARIABLE return_code)
if (return_code EQUAL 0)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
endforeach()
else()
endif ()
endforeach ()
else ()
set(_LOOP_STATUS OFF)
endif()
endforeach()
endif ()
endforeach ()
if(_MATCH_ALL)
if (_MATCH_ALL)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
endif ()
math(EXPR GIT_LOG_SKIP_NUM "${GIT_LOG_SKIP_NUM} + 1")
endwhile(_LOOP_STATUS)
else()
endwhile (_LOOP_STATUS)
else ()
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
endif()
else()
set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
endif ()
else ()
message(FATAL_ERROR "The directory ${working_dir} does not exist")
endif()
endif ()
endfunction()
# Define a function that extracts a cached package
......@@ -83,15 +83,15 @@ endfunction()
# Define a function that to create a new cached package
function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path)
if(EXISTS ${package_file})
if (EXISTS ${package_file})
message(STATUS "Removing existing package file: ${package_file}")
file(REMOVE ${package_file})
endif()
endif ()
string(REGEX REPLACE "(.+)/.+$" "\\1" package_dir ${package_file})
if(NOT EXISTS ${package_dir})
if (NOT EXISTS ${package_dir})
file(MAKE_DIRECTORY ${package_dir})
endif()
endif ()
message(STATUS "Will create cached package file: ${package_file}")
......@@ -116,89 +116,89 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
if (ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
endif ()
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
if(NOT ARG_STATIC_LIB)
if (ARG_STATIC_LIB AND ARG_SHARED_LIB)
if (NOT ARG_STATIC_LIB)
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endif ()
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
endif ()
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
if (WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
else ()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
endif ()
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_STATIC_LIB)
endif ()
elseif (ARG_STATIC_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
endif ()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_SHARED_LIB)
endif ()
elseif (ARG_SHARED_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
if (WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
else ()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
endif ()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
if (ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
if(ARG_INCLUDE_DIRECTORIES)
endif ()
if (ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
else()
endif ()
else ()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endif ()
endfunction()
core/src/index/cmake/DefineOptionsCore.cmake
浏览文件 @ bede7775
......@@ -13,16 +13,16 @@ macro(define_option name description default)
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
if ("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
endif ()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
foreach (item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
endforeach ()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
......@@ -35,19 +35,19 @@ macro(define_option_string name description default)
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
if(MILVUS_CPU_VERSION)
if (MILVUS_CPU_VERSION)
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
else()
else ()
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
endif()
endif ()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
......@@ -55,11 +55,11 @@ set_option_category("Thirdparty")
set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO")
define_option_string(KNOWHERE_DEPENDENCY_SOURCE
"Method to use for acquiring KNOWHERE's build dependencies"
"${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
"Method to use for acquiring KNOWHERE's build dependencies"
"${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(KNOWHERE_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
......@@ -82,7 +82,7 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
if(MSVC)
if (MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
......@@ -90,16 +90,16 @@ if(MSVC)
OFF)
define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
endif()
endif ()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
if (BUILD_UNIT_TEST)
define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" ON)
else()
else ()
define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
endif (BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
......@@ -111,12 +111,12 @@ macro(config_summary)
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${INDEX_BINARY_DIR}/compile_commands.json")
endif()
endif ()
foreach(category ${KNOWHERE_OPTION_CATEGORIES})
foreach (category ${KNOWHERE_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
......@@ -124,50 +124,50 @@ macro(config_summary)
set(option_names ${KNOWHERE_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
foreach (name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
if (${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
endif ()
endforeach ()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
foreach (name ${option_names})
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
else ()
set(value "${${name}}")
endif()
endif ()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
if (${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
else ()
set(description_padding "
")
endif()
endif ()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
if ("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
endif ()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
endif ()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach ()
endforeach()
endforeach ()
endmacro()
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp
浏览文件 @ bede7775
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "IndexGPUIDMAP.h"
#include <faiss/AutoTune.h>
......@@ -18,96 +35,95 @@
namespace knowhere {
VectorIndexPtr
GPUIDMAP::CopyGpuToCpu(const Config &config) {
std::lock_guard<std::mutex> lk(mutex_);
faiss::Index *device_index = index_.get();
faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
VectorIndexPtr
GPUIDMAP::CopyGpuToCpu(const Config& config) {
std::lock_guard<std::mutex> lk(mutex_);
std::shared_ptr<faiss::Index> new_index;
new_index.reset(host_index);
return std::make_shared<IDMAP>(new_index);
}
faiss::Index* device_index = index_.get();
faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index);
VectorIndexPtr
GPUIDMAP::Clone() {
auto cpu_idx = CopyGpuToCpu(Config());
std::shared_ptr<faiss::Index> new_index;
new_index.reset(host_index);
return std::make_shared<IDMAP>(new_index);
}
if (auto idmap = std::dynamic_pointer_cast<IDMAP>(cpu_idx)) {
return idmap->CopyCpuToGpu(gpu_id_, Config());
} else {
KNOWHERE_THROW_MSG("IndexType not Support GpuClone");
}
}
VectorIndexPtr
GPUIDMAP::Clone() {
auto cpu_idx = CopyGpuToCpu(Config());
BinarySet
GPUIDMAP::SerializeImpl() {
try {
MemoryIOWriter writer;
{
faiss::Index *index = index_.get();
faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(index);
faiss::write_index(host_index, &writer);
delete host_index;
}
auto data = std::make_shared<uint8_t>();
data.reset(writer.data_);
BinarySet res_set;
res_set.Append("IVF", data, writer.rp);
return res_set;
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
if (auto idmap = std::dynamic_pointer_cast<IDMAP>(cpu_idx)) {
return idmap->CopyCpuToGpu(gpu_id_, Config());
} else {
KNOWHERE_THROW_MSG("IndexType not Support GpuClone");
}
}
void
GPUIDMAP::LoadImpl(const BinarySet &index_binary) {
auto binary = index_binary.GetByName("IVF");
MemoryIOReader reader;
BinarySet
GPUIDMAP::SerializeImpl() {
try {
MemoryIOWriter writer;
{
reader.total = binary->size;
reader.data_ = binary->data.get();
faiss::Index *index = faiss::read_index(&reader);
faiss::Index* index = index_.get();
faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index);
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) {
ResScope rs(res, gpu_id_, false);
auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index);
index_.reset(device_index);
res_ = res;
} else {
KNOWHERE_THROW_MSG("Load error, can't get gpu resource");
}
delete index;
faiss::write_index(host_index, &writer);
delete host_index;
}
}
VectorIndexPtr
GPUIDMAP::CopyGpuToGpu(const int64_t &device_id, const Config &config) {
auto cpu_index = CopyGpuToCpu(config);
return std::static_pointer_cast<IDMAP>(cpu_index)->CopyCpuToGpu(device_id, config);
}
auto data = std::make_shared<uint8_t>();
data.reset(writer.data_);
float *
GPUIDMAP::GetRawVectors() {
KNOWHERE_THROW_MSG("Not support");
}
BinarySet res_set;
res_set.Append("IVF", data, writer.rp);
int64_t *
GPUIDMAP::GetRawIds() {
KNOWHERE_THROW_MSG("Not support");
return res_set;
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
void
GPUIDMAP::LoadImpl(const BinarySet& index_binary) {
auto binary = index_binary.GetByName("IVF");
MemoryIOReader reader;
{
reader.total = binary->size;
reader.data_ = binary->data.get();
faiss::Index* index = faiss::read_index(&reader);
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) {
ResScope rs(res, gpu_id_, false);
auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index);
index_.reset(device_index);
res_ = res;
} else {
KNOWHERE_THROW_MSG("Load error, can't get gpu resource");
}
void
GPUIDMAP::search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels,
const Config &cfg) {
ResScope rs(res_, gpu_id_);
index_->search(n, (float *) data, k, distances, labels);
delete index;
}
} // knowhere
}
VectorIndexPtr
GPUIDMAP::CopyGpuToGpu(const int64_t& device_id, const Config& config) {
auto cpu_index = CopyGpuToCpu(config);
return std::static_pointer_cast<IDMAP>(cpu_index)->CopyCpuToGpu(device_id, config);
}
float*
GPUIDMAP::GetRawVectors() {
KNOWHERE_THROW_MSG("Not support");
}
int64_t*
GPUIDMAP::GetRawIds() {
KNOWHERE_THROW_MSG("Not support");
}
void
GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
ResScope rs(res_, gpu_id_);
index_->search(n, (float*)data, k, distances, labels);
}
} // namespace knowhere
core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h
浏览文件 @ bede7775
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "IndexGPUIVF.h"
#include "IndexIVF.h"
#include "IndexIDMAP.h"
#include "IndexIVF.h"
#include <memory>
#include <utility>
namespace knowhere {
class GPUIDMAP : public IDMAP, public GPUIndex {
public:
explicit GPUIDMAP(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &res)
: IDMAP(std::move(index)), GPUIndex(device_id, res) {
}
class GPUIDMAP : public IDMAP, public GPUIndex {
public:
explicit GPUIDMAP(std::shared_ptr<faiss::Index> index, const int64_t& device_id, ResPtr& res)
: IDMAP(std::move(index)), GPUIndex(device_id, res) {
}
VectorIndexPtr
CopyGpuToCpu(const Config &config) override;
VectorIndexPtr
CopyGpuToCpu(const Config& config) override;
float *
GetRawVectors() override;
float*
GetRawVectors() override;
int64_t *
GetRawIds() override;
int64_t*
GetRawIds() override;
VectorIndexPtr
Clone() override;
VectorIndexPtr
Clone() override;
VectorIndexPtr
CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
VectorIndexPtr
CopyGpuToGpu(const int64_t& device_id, const Config& config) override;
protected:
void
search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels,
const Config &cfg) override;
protected:
void
search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override;
BinarySet
SerializeImpl() override;
BinarySet
SerializeImpl() override;
void
LoadImpl(const BinarySet &index_binary) override;
};
void
LoadImpl(const BinarySet& index_binary) override;
};
using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>;
using GPUIDMAPPtr = std::shared_ptr<GPUIDMAP>;
} // knowhere
\ No newline at end of file
} // namespace knowhere
\ No newline at end of file
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
浏览文件 @ bede7775
......@@ -18,9 +18,9 @@
#include <faiss/IndexFlat.h>
#include <faiss/MetaIndexes.h>
#include <faiss/index_factory.h>
#include <faiss/clone_index.h>
#include <faiss/AutoTune.h>
#include <faiss/clone_index.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#ifdef MILVUS_GPU_VERSION
......@@ -38,165 +38,163 @@
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#endif
namespace knowhere {
BinarySet
IDMAP::Serialize() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
std::lock_guard<std::mutex> lk(mutex_);
return SerializeImpl();
BinarySet
IDMAP::Serialize() {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
void
IDMAP::Load(const BinarySet &index_binary) {
std::lock_guard<std::mutex> lk(mutex_);
LoadImpl(index_binary);
std::lock_guard<std::mutex> lk(mutex_);
return SerializeImpl();
}
void
IDMAP::Load(const BinarySet& index_binary) {
std::lock_guard<std::mutex> lk(mutex_);
LoadImpl(index_binary);
}
DatasetPtr
IDMAP::Search(const DatasetPtr& dataset, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
DatasetPtr
IDMAP::Search(const DatasetPtr &dataset, const Config &config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
config->CheckValid();
// auto metric_type = config["metric_type"].as_string() == "L2" ?
// faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
// index_->metric_type = metric_type;
config->CheckValid();
// auto metric_type = config["metric_type"].as_string() == "L2" ?
// faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
// index_->metric_type = metric_type;
GETTENSOR(dataset)
GETTENSOR(dataset)
auto elems = rows * config->k;
auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems);
auto res_dis = (float*)malloc(sizeof(float) * elems);
auto elems = rows * config->k;
auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems);
auto res_dis = (float *) malloc(sizeof(float) * elems);
search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config());
search_impl(rows, (float *) p_data, config->k, res_dis, res_ids, Config());
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems);
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>(array, nullptr);
}
return std::make_shared<Dataset>(array, nullptr);
}
void
IDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {
index_->search(n, (float*)data, k, distances, labels);
}
void
IDMAP::search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels, const Config &cfg) {
index_->search(n, (float *) data, k, distances, labels);
void
IDMAP::Add(const DatasetPtr& dataset, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
void
IDMAP::Add(const DatasetPtr &dataset, const Config &config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset)
std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset)
// TODO: magic here.
auto array = dataset->array()[0];
auto p_ids = array->data()->GetValues<int64_t>(1, 0);
// TODO: magic here.
auto array = dataset->array()[0];
auto p_ids = array->data()->GetValues<int64_t>(1, 0);
index_->add_with_ids(rows, (float*)p_data, p_ids);
}
index_->add_with_ids(rows, (float *) p_data, p_ids);
}
int64_t
IDMAP::Count() {
return index_->ntotal;
}
int64_t
IDMAP::Count() {
return index_->ntotal;
}
int64_t
IDMAP::Dimension() {
return index_->d;
}
int64_t
IDMAP::Dimension() {
return index_->d;
}
// TODO(linxj): return const pointer
float *
IDMAP::GetRawVectors() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap *>(index_.get());
auto flat_index = dynamic_cast<faiss::IndexFlat *>(file_index->index);
return flat_index->xb.data();
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
float*
IDMAP::GetRawVectors() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap*>(index_.get());
auto flat_index = dynamic_cast<faiss::IndexFlat*>(file_index->index);
return flat_index->xb.data();
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
// TODO(linxj): return const pointer
int64_t *
IDMAP::GetRawIds() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap *>(index_.get());
return file_index->id_map.data();
} catch (std::exception &e) {
KNOWHERE_THROW_MSG(e.what());
}
int64_t*
IDMAP::GetRawIds() {
try {
auto file_index = dynamic_cast<faiss::IndexIDMap*>(index_.get());
return file_index->id_map.data();
} catch (std::exception& e) {
KNOWHERE_THROW_MSG(e.what());
}
}
const char *type = "IDMap,Flat";
const char* type = "IDMap,Flat";
void
IDMAP::Train(const Config &config) {
config->CheckValid();
auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type));
index_.reset(index);
}
void
IDMAP::Train(const Config& config) {
config->CheckValid();
VectorIndexPtr
IDMAP::Clone() {
std::lock_guard<std::mutex> lk(mutex_);
auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type));
index_.reset(index);
}
auto clone_index = faiss::clone_index(index_.get());
std::shared_ptr<faiss::Index> new_index;
new_index.reset(clone_index);
return std::make_shared<IDMAP>(new_index);
}
VectorIndexPtr
IDMAP::Clone() {
std::lock_guard<std::mutex> lk(mutex_);
VectorIndexPtr
IDMAP::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
auto clone_index = faiss::clone_index(index_.get());
std::shared_ptr<faiss::Index> new_index;
new_index.reset(clone_index);
return std::make_shared<IDMAP>(new_index);
}
VectorIndexPtr
IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());
std::shared_ptr<faiss::Index> device_index;
device_index.reset(gpu_index);
return std::make_shared<GPUIDMAP>(device_index, device_id, res);
} else {
KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
}
std::shared_ptr<faiss::Index> device_index;
device_index.reset(gpu_index);
return std::make_shared<GPUIDMAP>(device_index, device_id, res);
} else {
KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
}
#else
KNOWHERE_THROW_MSG("Calling IDMAP::CopyCpuToGpu when we are using CPU version");
KNOWHERE_THROW_MSG("Calling IDMAP::CopyCpuToGpu when we are using CPU version");
#endif
}
}
void
IDMAP::Seal() {
// do nothing
}
void
IDMAP::Seal() {
// do nothing
}
} // namespace knowhere
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
浏览文件 @ bede7775
......@@ -21,9 +21,9 @@
#include <faiss/IndexIVF.h>
#include <faiss/IndexIVFFlat.h>
#include <faiss/IndexIVFPQ.h>
#include <faiss/index_io.h>
#include <faiss/index_factory.h>
#include <faiss/clone_index.h>
#include <faiss/index_factory.h>
#include <faiss/index_io.h>
#ifdef MILVUS_GPU_VERSION
#include <faiss/gpu/GpuAutoTune.h>
#include <faiss/gpu/GpuCloner.h>
......@@ -240,7 +240,6 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6
VectorIndexPtr
IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
......
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
浏览文件 @ bede7775
......@@ -62,7 +62,6 @@ IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index>& index) {
VectorIndexPtr
IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
......
core/src/index/unittest/Helper.h
浏览文件 @ bede7775
......@@ -39,7 +39,7 @@ constexpr int64_t TEMPMEM = 1024 * 1024 * 300;
constexpr int64_t RESNUM = 2;
knowhere::IVFIndexPtr
IndexFactory(const std::string &type) {
IndexFactory(const std::string& type) {
if (type == "IVF") {
return std::make_shared<knowhere::IVF>();
} else if (type == "IVFPQ") {
......@@ -67,15 +67,15 @@ enum class ParameterType {
};
class ParamGenerator {
public:
static ParamGenerator &
public:
static ParamGenerator&
GetInstance() {
static ParamGenerator instance;
return instance;
}
knowhere::Config
Gen(const ParameterType &type) {
Gen(const ParameterType& type) {
if (type == ParameterType::ivf) {
auto tempconf = std::make_shared<knowhere::IVFCfg>();
tempconf->d = DIM;
......@@ -113,7 +113,7 @@ public:
#include <gtest/gtest.h>
class TestGpuIndexBase : public ::testing::Test {
protected:
protected:
void
SetUp() override {
#ifdef MILVUS_GPU_VERSION
......
core/src/index/unittest/test_idmap.cpp
浏览文件 @ bede7775
......@@ -22,8 +22,8 @@
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#endif
#include "Helper.h"
#include "unittest/utils.h"
......
core/src/index/unittest/test_ivf.cpp
浏览文件 @ bede7775
......@@ -47,7 +47,7 @@ using ::testing::TestWithParam;
using ::testing::Values;
class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, ParameterType>> {
protected:
protected:
void
SetUp() override {
#ifdef MILVUS_GPU_VERSION
......@@ -80,16 +80,15 @@ protected:
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
Values(
#ifdef MILVUS_GPU_VERSION
std::make_tuple("GPUIVF", ParameterType::ivf),
std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
std::make_tuple("GPUIVF", ParameterType::ivf),
std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),
std::make_tuple("GPUIVFSQ", ParameterType::ivfsq),
#ifdef CUSTOMIZATION
std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
#endif
#endif
std::make_tuple("IVF", ParameterType::ivf),
std::make_tuple("IVFPQ", ParameterType::ivfpq),
std::make_tuple("IVFSQ", ParameterType::ivfsq)));
std::make_tuple("IVF", ParameterType::ivf), std::make_tuple("IVFPQ", ParameterType::ivfpq),
std::make_tuple("IVFSQ", ParameterType::ivfsq)));
TEST_P(IVFTest, ivf_basic) {
assert(!xb.empty());
......@@ -109,9 +108,9 @@ TEST_P(IVFTest, ivf_basic) {
}
TEST_P(IVFTest, ivf_serialize) {
auto serialize = [](const std::string &filename, knowhere::BinaryPtr &bin, uint8_t *ret) {
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void *>(bin->data.get()), bin->size);
writer(static_cast<void*>(bin->data.get()), bin->size);
FileIOReader reader(filename);
reader(ret, bin->size);
......@@ -216,18 +215,18 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone G <=> C [" << index_type << "] success" << std::endl;
});
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone G <=> C [" << index_type << "] success" << std::endl;
});
} else {
EXPECT_THROW(
{
std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
},
knowhere::KnowhereException);
{
std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
},
knowhere::KnowhereException);
}
}
......@@ -241,18 +240,18 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
});
} else {
EXPECT_THROW(
{
std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
},
knowhere::KnowhereException);
{
std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl;
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, DEVICEID, knowhere::Config());
},
knowhere::KnowhereException);
}
}
}
......
core/src/index/unittest/test_nsg/CMakeLists.txt
浏览文件 @ bede7775
......@@ -19,9 +19,9 @@ set(interface_src
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp
)
if(NOT TARGET test_nsg)
if (NOT TARGET test_nsg)
add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs})
endif()
endif ()
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
##############################
......
core/src/main.cpp
浏览文件 @ bede7775
......@@ -32,7 +32,7 @@
INITIALIZE_EASYLOGGINGPP
void
print_help(const std::string &app_name) {
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl;
std::cout << " Options:" << std::endl;
std::cout << " -h --help Print this help" << std::endl;
......@@ -61,15 +61,15 @@ print_banner() {
}
int
main(int argc, char *argv[]) {
main(int argc, char* argv[]) {
print_banner();
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
{"log_conf_file", required_argument, nullptr, 'l'},
{"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, nullptr, 'p'},
{nullptr, 0, nullptr, 0}};
{"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, nullptr, 'p'},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
int64_t start_daemonized = 0;
......@@ -78,7 +78,7 @@ main(int argc, char *argv[]) {
std::string pid_filename;
std::string app_name = argv[0];
milvus::server::Server &server = milvus::server::Server::GetInstance();
milvus::server::Server& server = milvus::server::Server::GetInstance();
milvus::Status s;
if (argc < 2) {
......@@ -90,21 +90,21 @@ main(int argc, char *argv[]) {
while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) {
switch (value) {
case 'c': {
char *config_filename_ptr = strdup(optarg);
char* config_filename_ptr = strdup(optarg);
config_filename = config_filename_ptr;
free(config_filename_ptr);
std::cout << "Loading configuration from: " << config_filename << std::endl;
break;
}
case 'l': {
char *log_filename_ptr = strdup(optarg);
char* log_filename_ptr = strdup(optarg);
log_config_file = log_filename_ptr;
free(log_filename_ptr);
std::cout << "Initializing log config from: " << log_config_file << std::endl;
break;
}
case 'p': {
char *pid_filename_ptr = strdup(optarg);
char* pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
free(pid_filename_ptr);
std::cout << pid_filename << std::endl;
......@@ -147,7 +147,7 @@ main(int argc, char *argv[]) {
return EXIT_SUCCESS;
FAIL:
FAIL:
std::cout << "Milvus server exit..." << std::endl;
return EXIT_FAILURE;
}
core/src/metrics/SystemInfo.cpp
浏览文件 @ bede7775
......@@ -214,7 +214,6 @@ SystemInfo::CPUPercent() {
std::vector<uint64_t>
SystemInfo::GPUMemoryTotal() {
// get GPU usage percent
if (!initialized_)
Init();
......
core/src/scheduler/ResourceMgr.cpp
浏览文件 @ bede7775
......@@ -79,7 +79,9 @@ ResourceMgr::Add(ResourcePtr&& resource) {
gpu_resources_.emplace_back(ResourceWPtr(resource));
break;
}
default: { break; }
default: {
break;
}
}
resources_.emplace_back(resource);
......
core/src/scheduler/Scheduler.cpp
浏览文件 @ bede7775
......@@ -123,7 +123,9 @@ Scheduler::OnLoadCompleted(const EventPtr& event) {
Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_, resource);
break;
}
default: { break; }
default: {
break;
}
}
resource->WakeupLoader();
}
......
core/src/scheduler/resource/Resource.cpp
浏览文件 @ bede7775
......@@ -44,7 +44,9 @@ ToString(ResourceType type) {
case ResourceType::GPU: {
return "GPU";
}
default: { return "UNKNOWN"; }
default: {
return "UNKNOWN";
}
}
}
......
core/src/server/Config.cpp
浏览文件 @ bede7775
......@@ -709,7 +709,7 @@ CheckResource(const std::string& value) {
int32_t gpu_index = std::stoi(s.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
......@@ -1019,8 +1019,7 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
if (str != "cpu") {
value = std::stoi(str.substr(3));
}
else {
} else {
value = -1;
}
......
core/src/utils/BlockingQueue.inl
浏览文件 @ bede7775
......@@ -15,32 +15,26 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
namespace milvus {
namespace server {
template<typename T>
template <typename T>
void
BlockingQueue<T>::Put(const T &task) {
BlockingQueue<T>::Put(const T& task) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] {
return (queue_.size() < capacity_);
});
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
queue_.push(task);
empty_.notify_all();
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Take() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
queue_.pop();
......@@ -48,51 +42,45 @@ BlockingQueue<T>::Take() {
return front;
}
template<typename T>
template <typename T>
size_t
BlockingQueue<T>::Size() {
std::lock_guard<std::mutex> lock(mtx);
return queue_.size();
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Front() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
return front;
}
template<typename T>
template <typename T>
T
BlockingQueue<T>::Back() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] {
return !queue_.empty();
});
empty_.wait(lock, [this] { return !queue_.empty(); });
T back(queue_.back());
return back;
}
template<typename T>
template <typename T>
bool
BlockingQueue<T>::Empty() {
std::unique_lock<std::mutex> lock(mtx);
return queue_.empty();
}
template<typename T>
template <typename T>
void
BlockingQueue<T>::SetCapacity(const size_t capacity) {
capacity_ = (capacity > 0 ? capacity : capacity_);
}
} // namespace server
} // namespace milvus
} // namespace server
} // namespace milvus
core/src/utils/ValidationUtil.cpp
浏览文件 @ bede7775
......@@ -170,7 +170,6 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc
Status
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
......@@ -192,7 +191,6 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
Status
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
#ifdef MILVUS_GPU_VERSION
cudaDeviceProp deviceProp;
......
core/src/wrapper/KnowhereResource.cpp
浏览文件 @ bede7775
......@@ -34,7 +34,6 @@ constexpr int64_t M_BYTE = 1024 * 1024;
Status
KnowhereResource::Initialize() {
#ifdef MILVUS_GPU_VERSION
struct GpuResourceSetting {
......
core/src/wrapper/VecImpl.cpp
浏览文件 @ bede7775
......@@ -24,8 +24,8 @@
#ifdef MILVUS_GPU_VERSION
#include <src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.h>
#include <src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h>
#include <src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.h>
#endif
......@@ -35,216 +35,214 @@
*/
namespace milvus {
namespace engine {
Status
VecIndexImpl::BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg,
const int64_t &nt,
const float *xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
index_->set_index_model(model);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) {
try {
auto k = cfg->k;
auto dataset = GenDataset(nq, dim, xq);
Config search_cfg = cfg;
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
// auto& dists = dis_array;
// std::stringstream ss_id;
// std::stringstream ss_dist;
// for (auto i = 0; i < 10; i++) {
// for (auto j = 0; j < k; ++j) {
// ss_id << *(ids->data()->GetValues<int64_t>(1, i * k + j)) << " ";
// ss_dist << *(dists->data()->GetValues<float>(1, i * k + j)) << " ";
// }
// ss_id << std::endl;
// ss_dist << std::endl;
// }
// std::cout << "id\n" << ss_id.str() << std::endl;
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
memcpy(dist, p_dist, sizeof(float) * nq * k);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
knowhere::BinarySet
VecIndexImpl::Serialize() {
type = ConvertToCpuIndexType(type);
return index_->Serialize();
}
Status
VecIndexImpl::Load(const knowhere::BinarySet &index_binary) {
index_->Load(index_binary);
dim = Dimension();
return Status::OK();
}
int64_t
VecIndexImpl::Dimension() {
return index_->Dimension();
}
int64_t
VecIndexImpl::Count() {
return index_->Count();
}
IndexType
VecIndexImpl::GetType() {
return type;
}
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) {
// TODO(linxj): exception handle
namespace engine {
Status
VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
index_->set_preprocessor(preprocessor);
auto model = index_->Train(dataset, cfg);
index_->set_index_model(model);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) {
try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
Status
VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg) {
try {
auto k = cfg->k;
auto dataset = GenDataset(nq, dim, xq);
Config search_cfg = cfg;
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
// auto& dists = dis_array;
// std::stringstream ss_id;
// std::stringstream ss_dist;
// for (auto i = 0; i < 10; i++) {
// for (auto j = 0; j < k; ++j) {
// ss_id << *(ids->data()->GetValues<int64_t>(1, i * k + j)) << " ";
// ss_dist << *(dists->data()->GetValues<float>(1, i * k + j)) << " ";
// }
// ss_id << std::endl;
// ss_dist << std::endl;
// }
// std::cout << "id\n" << ss_id.str() << std::endl;
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
memcpy(dist, p_dist, sizeof(float) * nq * k);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
knowhere::BinarySet
VecIndexImpl::Serialize() {
type = ConvertToCpuIndexType(type);
return index_->Serialize();
}
Status
VecIndexImpl::Load(const knowhere::BinarySet& index_binary) {
index_->Load(index_binary);
dim = Dimension();
return Status::OK();
}
int64_t
VecIndexImpl::Dimension() {
return index_->Dimension();
}
int64_t
VecIndexImpl::Count() {
return index_->Count();
}
IndexType
VecIndexImpl::GetType() {
return type;
}
VecIndexPtr
VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) {
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
new_index->dim = dim;
return new_index;
auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
new_index->dim = dim;
return new_index;
#else
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToGpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToGpu when we are using CPU version");
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToGpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToGpu when we are using CPU version");
#endif
}
}
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config &cfg) {
// TODO(linxj): exception handle
VecIndexPtr
VecIndexImpl::CopyToCpu(const Config& cfg) {
// TODO(linxj): exception handle
#ifdef MILVUS_GPU_VERSION
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
new_index->dim = dim;
return new_index;
auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg);
auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
new_index->dim = dim;
return new_index;
#else
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToCpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToCpu when we are using CPU version");
WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToCpu when we are using CPU version";
throw WrapperException("Calling VecIndexImpl::CopyToCpu when we are using CPU version");
#endif
}
VecIndexPtr
VecIndexImpl::Clone() {
// TODO(linxj): exception handle
auto clone_index = std::make_shared<VecIndexImpl>(index_->Clone(), type);
clone_index->dim = dim;
return clone_index;
}
int64_t
VecIndexImpl::GetDeviceId() {
}
VecIndexPtr
VecIndexImpl::Clone() {
// TODO(linxj): exception handle
auto clone_index = std::make_shared<VecIndexImpl>(index_->Clone(), type);
clone_index->dim = dim;
return clone_index;
}
int64_t
VecIndexImpl::GetDeviceId() {
#ifdef MILVUS_GPU_VERSION
if (auto device_idx = std::dynamic_pointer_cast<knowhere::GPUIndex>(index_)) {
return device_idx->GetGpuDevice();
}
if (auto device_idx = std::dynamic_pointer_cast<knowhere::GPUIndex>(index_)) {
return device_idx->GetGpuDevice();
}
#else
// else
return -1; // -1 == cpu
// else
return -1; // -1 == cpu
#endif
}
float *
BFIndex::GetRawVectors() {
auto raw_index = std::dynamic_pointer_cast<knowhere::IDMAP>(index_);
if (raw_index) {
return raw_index->GetRawVectors();
}
return nullptr;
}
int64_t *
BFIndex::GetRawIds() {
return std::static_pointer_cast<knowhere::IDMAP>(index_)->GetRawIds();
}
ErrorCode
BFIndex::Build(const Config &cfg) {
try {
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_UNEXPECTED_ERROR;
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_ERROR;
}
return KNOWHERE_SUCCESS;
}
Status
BFIndex::BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg, const int64_t &nt,
const float *xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
} // namespace engine
}
float*
BFIndex::GetRawVectors() {
auto raw_index = std::dynamic_pointer_cast<knowhere::IDMAP>(index_);
if (raw_index) {
return raw_index->GetRawVectors();
}
return nullptr;
}
int64_t*
BFIndex::GetRawIds() {
return std::static_pointer_cast<knowhere::IDMAP>(index_)->GetRawIds();
}
ErrorCode
BFIndex::Build(const Config& cfg) {
try {
dim = cfg->d;
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_UNEXPECTED_ERROR;
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return KNOWHERE_ERROR;
}
return KNOWHERE_SUCCESS;
}
Status
BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) {
try {
dim = cfg->d;
auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
std::static_pointer_cast<knowhere::IDMAP>(index_)->Train(cfg);
index_->Add(dataset, cfg);
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
} catch (std::exception& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_ERROR, e.what());
}
return Status::OK();
}
} // namespace engine
} // namespace milvus
core/src/wrapper/VecIndex.cpp
浏览文件 @ bede7775
......@@ -28,12 +28,12 @@
#ifdef MILVUS_GPU_VERSION
#include <cuda.h>
#include "wrapper/gpu/GPUVecImpl.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "wrapper/gpu/GPUVecImpl.h"
#endif
namespace milvus {
......@@ -168,7 +168,9 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::NSG>(gpu_device);
break;
}
default: { return nullptr; }
default: {
return nullptr;
}
}
return std::make_shared<VecIndexImpl>(index, type);
}
......@@ -276,7 +278,9 @@ ConvertToCpuIndexType(const IndexType& type) {
case IndexType::FAISS_IVFSQ8_MIX: {
return IndexType::FAISS_IVFSQ8_CPU;
}
default: { return type; }
default: {
return type;
}
}
}
......@@ -291,7 +295,9 @@ ConvertToGpuIndexType(const IndexType& type) {
case IndexType::FAISS_IVFSQ8_CPU: {
return IndexType::FAISS_IVFSQ8_GPU;
}
default: { return type; }
default: {
return type;
}
}
}
......
core/src/wrapper/WrapperException.cpp
浏览文件 @ bede7775
......@@ -20,9 +20,11 @@
namespace milvus {
namespace engine {
WrapperException::WrapperException(const std::string &msg) : msg(msg) {}
WrapperException::WrapperException(const std::string& msg) : msg(msg) {
}
const char *WrapperException::what() const noexcept {
const char*
WrapperException::what() const noexcept {
return msg.c_str();
}
......
core/src/wrapper/WrapperException.h
浏览文件 @ bede7775
......@@ -27,7 +27,8 @@ class WrapperException : public std::exception {
public:
explicit WrapperException(const std::string& msg);
const char* what() const noexcept override;
const char*
what() const noexcept override;
const std::string msg;
};
......
core/src/wrapper/gpu/GPUVecImpl.cpp
浏览文件 @ bede7775
......@@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "wrapper/VecImpl.h"
#include "GPUVecImpl.h"
#include "src/wrapper/DataTransfer.h"
#include "knowhere/common/Exception.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexGPUIDMAP.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "knowhere/index/vector_index/IndexIDMAP.h"
#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"
#include "knowhere/index/vector_index/helpers/Cloner.h"
#include "src/wrapper/DataTransfer.h"
#include "utils/Log.h"
#include "wrapper/VecImpl.h"
/*
* no parameter check in this layer.
......@@ -34,7 +34,6 @@
namespace milvus {
namespace engine {
// TODO(linxj): add lock here.
Status
IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
......
core/src/wrapper/gpu/GPUVecImpl.h
浏览文件 @ bede7775
......@@ -20,46 +20,47 @@
#include <memory>
#include <utility>
#include "wrapper/VecIndex.h"
#include "knowhere/index/vector_index/VectorIndex.h"
#include "wrapper/VecImpl.h"
#include "wrapper/VecIndex.h"
namespace milvus {
namespace engine {
class IVFMixIndex : public VecIndexImpl {
public:
explicit IVFMixIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType &type)
: VecIndexImpl(std::move(index), type) {
public:
explicit IVFMixIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType& type)
: VecIndexImpl(std::move(index), type) {
}
Status
BuildAll(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg, const int64_t &nt,
const float *xt) override;
BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt,
const float* xt) override;
Status
Load(const knowhere::BinarySet &index_binary) override;
Load(const knowhere::BinarySet& index_binary) override;
};
class IVFHybridIndex : public IVFMixIndex {
public:
explicit IVFHybridIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType &type)
: IVFMixIndex(std::move(index), type) {
public:
explicit IVFHybridIndex(std::shared_ptr<knowhere::VectorIndex> index, const IndexType& type)
: IVFMixIndex(std::move(index), type) {
}
knowhere::QuantizerPtr
LoadQuantizer(const Config &conf) override;
LoadQuantizer(const Config& conf) override;
Status
SetQuantizer(const knowhere::QuantizerPtr &q) override;
SetQuantizer(const knowhere::QuantizerPtr& q) override;
Status
UnsetQuantizer() override;
std::pair<VecIndexPtr, knowhere::QuantizerPtr>
CopyToGpuWithQuantizer(const int64_t &device_id, const Config &cfg) override;
CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) override;
VecIndexPtr
LoadData(const knowhere::QuantizerPtr &q, const Config &conf) override;
LoadData(const knowhere::QuantizerPtr& q, const Config& conf) override;
};
} // namespace engine
......
core/unittest/CMakeLists.txt
浏览文件 @ bede7775
......@@ -17,9 +17,9 @@
# under the License.
#-------------------------------------------------------------------------------
foreach(dir ${INDEX_INCLUDE_DIRS})
foreach (dir ${INDEX_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
endforeach ()
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
......@@ -127,7 +127,7 @@ if (MILVUS_GPU_VERSION)
set(common_files ${common_files}
${wrapper_gpu_files}
)
endif()
endif ()
add_subdirectory(db)
add_subdirectory(wrapper)
......
core/unittest/db/test_db.cpp
浏览文件 @ bede7775
......@@ -15,25 +15,24 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "cache/CpuCacheMgr.h"
#include "db/Constants.h"
#include "db/DB.h"
#include "db/DBFactory.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include "db/DBFactory.h"
#include "cache/CpuCacheMgr.h"
#include "utils/CommonUtil.h"
#include "db/utils.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
#include <thread>
namespace {
static const char *TABLE_NAME = "test_group";
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
......@@ -49,10 +48,10 @@ BuildTableSchema() {
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
......@@ -68,16 +67,15 @@ CurrentTmDate(int64_t offset_day = 0) {
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
std::string str =
std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday);
return str;
}
void
ConvertTimeRangeToDBDates(const std::string &start_value,
const std::string &end_value,
std::vector<milvus::engine::meta::DateT> &dates) {
ConvertTimeRangeToDBDates(const std::string& start_value, const std::string& end_value,
std::vector<milvus::engine::meta::DateT>& dates) {
dates.clear();
time_t tt_start, tt_end;
......@@ -90,8 +88,7 @@ ConvertTimeRangeToDBDates(const std::string &start_value,
return;
}
int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) /
DAY_SECONDS;
int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / DAY_SECONDS;
if (days == 0) {
return;
}
......@@ -101,13 +98,12 @@ ConvertTimeRangeToDBDates(const std::string &start_value,
tm tm_day;
milvus::server::CommonUtil::ConvertTime(tt_day, tm_day);
int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 +
tm_day.tm_mday;//according to db logic
int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic
dates.push_back(date);
}
}
} // namespace
} // namespace
TEST_F(DBTest, CONFIG_TEST) {
{
......@@ -232,7 +228,7 @@ TEST_F(DBTest, DB_TEST) {
TEST_F(DBTest, SEARCH_TEST) {
std::string config_path(CONFIG_PATH);
config_path += CONFIG_FILE;
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(config_path);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
......@@ -266,22 +262,24 @@ TEST_F(DBTest, SEARCH_TEST) {
}
// result data
//std::vector<long> nns_gt(k*nq);
// std::vector<long> nns_gt(k*nq);
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
// std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
if (j == 200) {
sleep(1);
}
ASSERT_TRUE(stat.ok());
}
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
milvus::engine::QueryResults results;
......@@ -289,7 +287,7 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
{ // search by specify index file
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
milvus::engine::QueryResults results;
......@@ -298,9 +296,9 @@ TEST_F(DBTest, SEARCH_TEST) {
}
#ifdef CUSTOMIZATION
//test FAISS_IVFSQ8H optimizer
// test FAISS_IVFSQ8H optimizer
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
milvus::engine::QueryResults results;
......@@ -314,7 +312,7 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
{ // search by specify index file
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
milvus::engine::QueryResults results;
......@@ -322,7 +320,6 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
#endif
}
......@@ -348,8 +345,8 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
}
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
int64_t prev_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);
......@@ -415,12 +412,12 @@ TEST_F(DBTest, INDEX_TEST) {
ASSERT_EQ(vector_ids.size(), nb);
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int) milvus::engine::MetricType::IP;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int)milvus::engine::MetricType::IP;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFFLAT;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
......@@ -449,7 +446,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for (auto &schema : table_schema_array) {
for (auto& schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
......
core/unittest/db/test_db_mysql.cpp
浏览文件 @ bede7775
......@@ -15,21 +15,21 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/Constants.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include "db/utils.h"
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
#include <thread>
namespace {
static const char *TABLE_NAME = "test_group";
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
......@@ -39,22 +39,22 @@ BuildTableSchema() {
milvus::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
table_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
} // namespace
} // namespace
TEST_F(MySqlDBTest, DB_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
......@@ -102,10 +102,10 @@ TEST_F(MySqlDBTest, DB_TEST) {
ASSERT_TRUE(stat.ok());
for (auto k = 0; k < qb; ++k) {
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
bool exists = false;
for (auto &result : results[k]) {
for (auto& result : results[k]) {
if (result.first == target_ids[k]) {
exists = true;
}
......@@ -128,12 +128,12 @@ TEST_F(MySqlDBTest, DB_TEST) {
int loop = INSERT_LOOP;
for (auto i = 0; i < loop; ++i) {
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
// } else {
// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
// }
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
// } else {
// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
// }
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
......@@ -173,20 +173,22 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
}
// result data
//std::vector<long> nns_gt(k*nq);
// std::vector<long> nns_gt(k*nq);
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
// std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
if (j == 200) {
sleep(1);
}
ASSERT_TRUE(stat.ok());
}
sleep(2); // wait until build index finish
sleep(2); // wait until build index finish
milvus::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
......@@ -201,7 +203,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for (auto &schema : table_schema_array) {
for (auto& schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
......@@ -241,7 +243,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
TEST_F(MySqlDBTest, DELETE_TEST) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl;
// std::cout << stat.ToString() << std::endl;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
......@@ -267,14 +269,13 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
// std::vector<engine::meta::DateT> dates;
// stat = db_->DeleteTable(TABLE_NAME, dates);
//// std::cout << "5 sec start" << std::endl;
// std::this_thread::sleep_for(std::chrono::seconds(5));
//// std::cout << "5 sec finish" << std::endl;
// ASSERT_TRUE(stat.ok());
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
// std::vector<engine::meta::DateT> dates;
// stat = db_->DeleteTable(TABLE_NAME, dates);
//// std::cout << "5 sec start" << std::endl;
// std::this_thread::sleep_for(std::chrono::seconds(5));
//// std::cout << "5 sec finish" << std::endl;
// ASSERT_TRUE(stat.ok());
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
}
core/unittest/db/test_engine.cpp
浏览文件 @ bede7775
......@@ -26,55 +26,36 @@
TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::INVALID,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::INVALID, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr == nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IDMAP,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IDMAP, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
auto engine_ptr =
milvus::engine::EngineFactory::Build(512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::FAISS_IVFSQ8,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_IVFSQ8, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
milvus::engine::EngineType::NSG_MIX,
milvus::engine::MetricType::IP,
1024);
512, "/tmp/milvus_index_1", milvus::engine::EngineType::NSG_MIX, milvus::engine::MetricType::IP, 1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
......@@ -84,21 +65,17 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension,
file_path,
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
dimension, file_path, milvus::engine::EngineType::FAISS_IVFFLAT, milvus::engine::MetricType::IP, 1024);
std::vector<float> data;
std::vector<int64_t> ids;
const int row_count = 10000;
data.reserve(row_count*dimension);
data.reserve(row_count * dimension);
ids.reserve(row_count);
for (int64_t i = 0; i < row_count; i++) {
ids.push_back(i);
for (uint16_t k = 0; k < dimension; k++) {
data.push_back(i*dimension + k);
data.push_back(i * dimension + k);
}
}
......@@ -109,14 +86,14 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
ASSERT_EQ(engine_ptr->Count(), ids.size());
status = engine_ptr->CopyToGpu(0, false);
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
auto new_engine = engine_ptr->Clone();
ASSERT_EQ(new_engine->Dimension(), dimension);
ASSERT_EQ(new_engine->Count(), ids.size());
status = new_engine->CopyToCpu();
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
//ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
}
core/unittest/db/test_mem.cpp
浏览文件 @ bede7775
......@@ -15,25 +15,24 @@
// specific language governing permissions and limitations
// under the License.
#include "gtest/gtest.h"
#include "db/insert/VectorSource.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/MemTable.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "db/insert/MemTable.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/VectorSource.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "db/utils.h"
#include "metrics/Metrics.h"
#include <boost/filesystem.hpp>
#include <thread>
#include <chrono>
#include <cmath>
#include <fstream>
#include <iostream>
#include <cmath>
#include <random>
#include <chrono>
#include <thread>
namespace {
......@@ -42,8 +41,7 @@ static constexpr int64_t TABLE_DIM = 256;
std::string
GetTableName() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
static std::string table_name = std::to_string(micros);
return table_name;
}
......@@ -63,11 +61,10 @@ BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.resize(n * TABLE_DIM);
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++)
data[TABLE_DIM * i + j] = drand48();
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
}
}
} // namespace
} // namespace
TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
......@@ -86,12 +83,10 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
milvus::engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
milvus::engine::ExecutionEnginePtr execution_engine_ =
milvus::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(milvus::engine::EngineType)table_file_schema.engine_type_,
(milvus::engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
milvus::engine::ExecutionEnginePtr execution_engine_ = milvus::engine::EngineFactory::Build(
table_file_schema.dimension_, table_file_schema.location_,
(milvus::engine::EngineType)table_file_schema.engine_type_,
(milvus::engine::MetricType)table_file_schema.metric_type_, table_schema.nlist_);
milvus::engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
......@@ -129,7 +124,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
status = mem_table_file.Add(source, vector_ids);
ASSERT_TRUE(status.ok());
// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl;
// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl;
vector_ids = source->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
......@@ -141,8 +136,8 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
......@@ -163,8 +158,8 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
milvus::engine::VectorSourcePtr
source_100 = std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::VectorSourcePtr source_100 =
std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::MemTable mem_table(GetTableName(), impl_, options);
......@@ -184,8 +179,8 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr source_128M =
std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
......@@ -239,7 +234,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk
std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
std::random_device rd;
std::mt19937 gen(rd());
......@@ -400,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[0], nb);
ASSERT_TRUE(stat.ok());
nb = 262144; //512M
nb = 262144; // 512M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
......@@ -412,7 +407,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[0], nb / 2);
ASSERT_TRUE(stat.ok());
nb = 65536; //128M
nb = 65536; // 128M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
......@@ -432,4 +427,3 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ASSERT_EQ(vector_ids[i], i + nb);
}
}
core/unittest/db/test_meta.cpp
浏览文件 @ bede7775
......@@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/Constants.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/utils.h"
#include <gtest/gtest.h>
#include <thread>
#include <stdlib.h>
#include <time.h>
#include <thread>
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
......@@ -155,7 +155,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
......@@ -200,7 +200,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
......@@ -293,16 +293,13 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
......@@ -315,19 +312,15 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
......
core/unittest/db/test_meta_mysql.cpp
浏览文件 @ bede7775
......@@ -15,17 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "db/utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/utils.h"
#include <iostream>
#include <thread>
#include <stdlib.h>
#include <time.h>
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
#include <stdlib.h>
#include <time.h>
#include <iostream>
#include <thread>
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
......@@ -52,7 +52,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = "";
status = impl_->CreateTable(table);
// ASSERT_TRUE(status.ok());
// ASSERT_TRUE(status.ok());
status = impl_->DropAll();
ASSERT_TRUE(status.ok());
......@@ -79,8 +79,8 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(cnt, 0UL);
auto file_id = table_file.file_id_;
......@@ -159,7 +159,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
......@@ -167,7 +167,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
}
std::vector<int> file_types = {
(int) milvus::engine::meta::TableFileSchema::NEW,
(int)milvus::engine::meta::TableFileSchema::NEW,
};
std::vector<std::string> file_ids;
status = impl.FilesByType(table_id, file_types, file_ids);
......@@ -219,7 +219,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
for (auto& file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
......@@ -313,16 +313,13 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
......@@ -335,19 +332,15 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::NEW, milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX, milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX, milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
......
core/unittest/db/test_misc.cpp
浏览文件 @ bede7775
......@@ -16,15 +16,15 @@
// under the License.
#include "db/Options.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/engine/EngineFactory.h"
#include "db/Utils.h"
#include "utils/Status.h"
#include "db/engine/EngineFactory.h"
#include "db/meta/SqliteMetaImpl.h"
#include "utils/Exception.h"
#include "utils/Status.h"
#include <gtest/gtest.h>
#include <thread>
#include <boost/filesystem.hpp>
#include <thread>
#include <vector>
TEST(DBMiscTest, EXCEPTION_TEST) {
......@@ -40,7 +40,7 @@ TEST(DBMiscTest, EXCEPTION_TEST) {
TEST(DBMiscTest, OPTIONS_TEST) {
try {
milvus::engine::ArchiveConf archive("$$##");
} catch (std::exception &ex) {
} catch (std::exception& ex) {
ASSERT_TRUE(true);
}
......@@ -61,10 +61,7 @@ TEST(DBMiscTest, OPTIONS_TEST) {
{
milvus::engine::ArchiveConf archive("delete");
milvus::engine::ArchiveConf::CriteriaT criterial = {
{"disk", 1024},
{"days", 100}
};
milvus::engine::ArchiveConf::CriteriaT criterial = {{"disk", 1024}, {"days", 100}};
archive.SetCriterias(criterial);
auto crit = archive.GetCriterias();
......@@ -95,17 +92,17 @@ TEST(DBMiscTest, UTILS_TEST) {
auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
}
// options.slave_paths.push_back("/");
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
//
// options.path = "/";
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
// options.slave_paths.push_back("/");
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
//
// options.path = "/";
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
milvus::engine::meta::TableFileSchema file;
file.id_ = 50;
......
core/unittest/db/test_search.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/db/utils.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/db/utils.h
浏览文件 @ bede7775
此差异已折叠。
core/unittest/main.cpp
浏览文件 @ bede7775
......@@ -15,15 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "external/easyloggingpp/easylogging++.h"
INITIALIZE_EASYLOGGINGPP
int
main(int argc, char **argv) {
main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
core/unittest/metrics/test_metricbase.cpp
浏览文件 @ bede7775
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/Metrics.h"
#include <gtest/gtest.h>
......
core/unittest/metrics/test_metrics.cpp
浏览文件 @ bede7775
......@@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <chrono>
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <gtest/gtest.h>
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "metrics/Metrics.h"
#include "metrics/utils.h"
#include "server/Config.h"
TEST_F(MetricTest, METRIC_TEST) {
milvus::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
......@@ -36,15 +36,15 @@ TEST_F(MetricTest, METRIC_TEST) {
milvus::server::Metrics::GetInstance();
milvus::server::SystemInfo::GetInstance().Init();
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
milvus::server::Metrics::GetInstance().Init();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char *group_name = "test_group";
static const char* group_name = "test_group";
static const int group_dim = 256;
milvus::engine::meta::TableSchema group_info;
......@@ -61,14 +61,14 @@ TEST_F(MetricTest, METRIC_TEST) {
int d = 256;
int nb = 50;
float *xb = new float[d * nb];
float* xb = new float[d * nb];
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) xb[d * i + j] = drand48();
xb[d * i] += i / 2000.;
}
int qb = 5;
float *qxb = new float[d * qb];
float* qxb = new float[d * qb];
for (int i = 0; i < qb; i++) {
for (int j = 0; j < d; j++) qxb[d * i + j] = drand48();
qxb[d * i] += i / 2000.;
......@@ -90,17 +90,16 @@ TEST_F(MetricTest, METRIC_TEST) {
prev_count = count;
START_TIMER;
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count * group_dim * sizeof(float)) / (1024 * 1024) << " M";
for (auto k = 0; k < qb; ++k) {
// ASSERT_EQ(results[k][0].first, target_ids[k]);
// ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
}
ASSERT_TRUE(count >= prev_count);
std::this_thread::sleep_for(std::chrono::seconds(1));
......@@ -153,5 +152,3 @@ TEST_F(MetricTest, COLLECTOR_METRICS_TEST) {
milvus::server::MetricCollector metric_collector();
}
core/unittest/metrics/test_prometheus.cpp
浏览文件 @ bede7775
......@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "server/Config.h"
......
core/unittest/metrics/utils.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/metrics/utils.h
浏览文件 @ bede7775
此差异已折叠。
core/unittest/server/test_cache.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/server/test_rpc.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/server/test_util.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/server/utils.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/server/utils.h
浏览文件 @ bede7775
此差异已折叠。
core/unittest/wrapper/utils.cpp
浏览文件 @ bede7775
此差异已折叠。
core/unittest/wrapper/utils.h
浏览文件 @ bede7775
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或