未验证 提交 3eda414b 编写于 作者: J Jin Hai 提交者: GitHub

Merge pull request #240 from youny626/0.6.0

Change default compile option to CPU version & add MKL/OpenBlas message at server start & update config and unittest
......@@ -31,16 +31,16 @@ GET_CURRENT_TIME(BUILD_TIME)
string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message(STATUS "Build time = ${BUILD_TIME}")
MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if(GIT_BRANCH_NAME STREQUAL "")
if (GIT_BRANCH_NAME STREQUAL "")
execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif()
ENDMACRO (GET_GIT_BRANCH_NAME)
endif ()
ENDMACRO(GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
if(NOT GIT_BRANCH_NAME STREQUAL "")
if (NOT GIT_BRANCH_NAME STREQUAL "")
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
endif ()
......@@ -69,7 +69,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL "")
message(WARNING "Failed to determine Milvus version from git branch name")
set(MILVUS_VERSION "0.6.0")
endif()
endif ()
message(STATUS "Build version = ${MILVUS_VERSION}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY)
......@@ -118,17 +118,17 @@ include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
set(MILVUS_GPU_VERSION false)
if (MILVUS_CPU_VERSION)
message(STATUS "Building Milvus CPU version")
add_compile_definitions("MILVUS_CPU_VERSION")
else ()
set(MILVUS_CPU_VERSION false)
if (MILVUS_GPU_VERSION)
message(STATUS "Building Milvus GPU version")
set(MILVUS_GPU_VERSION true)
add_compile_definitions("MILVUS_GPU_VERSION")
enable_language(CUDA)
find_package(CUDA 10 REQUIRED)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
else ()
message(STATUS "Building Milvus CPU version")
set(MILVUS_CPU_VERSION true)
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
if (CMAKE_BUILD_TYPE STREQUAL "Release")
......
......@@ -12,7 +12,7 @@ USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
CPU_VERSION="OFF"
GPU_VERSION="OFF" #defaults to CPU version
WITH_MKL="OFF"
CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
......@@ -51,7 +51,7 @@ do
c)
BUILD_COVERAGE="ON"
;;
g)
z)
PROFILING="ON"
;;
j)
......@@ -60,8 +60,8 @@ do
x)
CUSTOMIZATION="OFF" # force use ori faiss
;;
z)
CPU_VERSION="ON"
g)
GPU_VERSION="ON"
;;
m)
WITH_MKL="ON"
......@@ -77,14 +77,14 @@ parameter:
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-g: profiling(default: OFF)
-z: profiling(default: OFF)
-j: use jfrog cache build directory(default: OFF)
-z: build pure CPU version(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-z] [-m] [-h]
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
"
exit 0
;;
......@@ -116,7 +116,7 @@ CMAKE_CMD="cmake \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
-DMILVUS_CPU_VERSION=${CPU_VERSION} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
../"
echo ${CMAKE_CMD}
......
......@@ -13,16 +13,16 @@ macro(define_option name description default)
endmacro()
function(list_join lst glue out)
if("${${lst}}" STREQUAL "")
if ("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif()
endif ()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach(item ${${lst}})
foreach (item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach()
endforeach ()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
......@@ -35,15 +35,15 @@ macro(define_option_string name description default)
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif()
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
set_option_category("GPU version")
define_option(MILVUS_CPU_VERSION "Build CPU version only" OFF)
define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
#----------------------------------------------------------------------
set_option_category("Thirdparty")
......@@ -75,33 +75,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif()
endif ()
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF)
endif()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
unset(MILVUS_BUILD_TESTS CACHE)
if (BUILD_UNIT_TEST)
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
else()
else ()
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
endif(BUILD_UNIT_TEST)
endif (BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
......@@ -113,12 +101,12 @@ macro(config_summary)
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS})
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
endif()
endif ()
foreach(category ${MILVUS_OPTION_CATEGORIES})
foreach (category ${MILVUS_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
......@@ -126,50 +114,50 @@ macro(config_summary)
set(option_names ${MILVUS_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach(name ${option_names})
foreach (name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if(${max_value_length} LESS ${value_length})
if (${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif()
endforeach()
endif ()
endforeach ()
foreach(name ${option_names})
if("${${name}_OPTION_TYPE}" STREQUAL "string")
foreach (name ${option_names})
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else()
else ()
set(value "${${name}}")
endif()
endif ()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if(${description_length} LESS 70)
if (${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else()
else ()
set(description_padding "
")
endif()
endif ()
set(comment "[${name}]")
if("${value}" STREQUAL "${default}")
if ("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif()
endif ()
if(NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif()
endif ()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach()
endforeach ()
endforeach()
endforeach ()
endmacro()
......@@ -32,9 +32,9 @@ cache_config:
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
# Skip the following config if you are using GPU version
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
#Uncomment the following config if you are using GPU version
# gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
# gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
engine_config:
use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
......@@ -44,5 +44,4 @@ engine_config:
resource_config:
search_resources: # define the devices used for search computation, must be in format: cpu or gpux
- cpu
- gpu0
index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu / gpux
index_build_device: cpu # CPU / GPU used for building index, must be in format: cpu / gpux
......@@ -24,6 +24,9 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
add_subdirectory(index)
if (BUILD_FAISS_WITH_MKL)
add_compile_definitions("WITH_MKL")
endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
foreach (dir ${INDEX_INCLUDE_DIRS})
......
......@@ -22,7 +22,7 @@ cmake_minimum_required(VERSION 3.14)
message(STATUS "------------------------------KNOWHERE-----------------------------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.5.0")
set(KNOWHERE_VERSION "0.6.0")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}")
project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 14)
......@@ -72,17 +72,17 @@ include(ExternalProject)
include(DefineOptionsCore)
include(BuildUtilsCore)
set(KNOWHERE_GPU_VERSION false)
if (MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION)
message(STATUS "Building Knowhere CPU version")
add_compile_definitions("MILVUS_CPU_VERSION")
else ()
set(KNOWHERE_CPU_VERSION false)
if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION)
message(STATUS "Building Knowhere GPU version")
add_compile_definitions("MILVUS_GPU_VERSION")
set(KNOWHERE_GPU_VERSION true)
enable_language(CUDA)
find_package(CUDA 10 REQUIRED)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
else ()
message(STATUS "Building Knowhere CPU version")
set(KNOWHERE_CPU_VERSION true)
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
include(ThirdPartyPackagesCore)
......
......@@ -41,12 +41,12 @@ macro(define_option_string name description default)
endmacro()
#----------------------------------------------------------------------
set_option_category("CPU version")
set_option_category("GPU version")
if (MILVUS_CPU_VERSION)
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" ON)
if (MILVUS_GPU_VERSION)
define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON)
else ()
define_option(KNOWHERE_CPU_VERSION "Build CPU version only" OFF)
define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF)
endif ()
#----------------------------------------------------------------------
......@@ -81,17 +81,6 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF)
#----------------------------------------------------------------------
if (MSVC)
set_option_category("MSVC")
define_option(MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF)
define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF)
endif ()
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
......
......@@ -51,7 +51,13 @@ print_banner() {
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
std::cout << std::endl;
std::cout << "Welcome to Milvus!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with "
#ifdef WITH_MKL
<< "MKL"
#else
<< "OpenBLAS"
#endif
<< " library." << std::endl;
#ifdef MILVUS_CPU_VERSION
std::cout << "You are using Milvus CPU version" << std::endl;
#else
......@@ -137,7 +143,7 @@ main(int argc, char* argv[]) {
s = server.Start();
if (s.ok()) {
std::cout << "Milvus server start successfully." << std::endl;
std::cout << "Milvus server started successfully!" << std::endl;
} else {
goto FAIL;
}
......
......@@ -25,6 +25,7 @@
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace milvus {
......@@ -343,6 +344,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
if (!s.ok()) {
return s;
}
s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
if (!s.ok()) {
return s;
......@@ -403,8 +409,7 @@ Status
Config::CheckServerConfigDeployMode(const std::string& value) {
if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") {
return Status(SERVER_INVALID_ARGUMENT,
"server_config.deploy_mode is not one of "
"single, cluster_readonly, and cluster_writable.");
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.");
}
return Status::OK();
}
......@@ -592,15 +597,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
int gpu_index;
Status s = GetResourceConfigIndexBuildDevice(gpu_index);
int device_id;
Status s = GetResourceConfigIndexBuildDevice(device_id);
if (!s.ok()) {
return s;
}
size_t gpu_memory;
if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index);
if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id);
return Status(SERVER_UNEXPECTED_ERROR, msg);
} else if (gpu_cache_capacity >= gpu_memory) {
std::string msg = "Invalid gpu cache capacity: " + value +
......@@ -689,29 +694,33 @@ Config::CheckResourceConfigMode(const std::string& value) {
}
Status
CheckGpuDevice(const std::string& value) {
CheckResource(const std::string& value) {
std::string s = value;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
#ifdef MILVUS_CPU_VERSION
if (s != "cpu") {
return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s);
}
#else
const std::regex pat("gpu(\\d+)");
std::cmatch m;
if (!std::regex_match(value.c_str(), m, pat)) {
std::string msg = "Invalid gpu device: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
const std::regex pat("cpu|gpu(\\d+)");
std::smatch m;
if (!std::regex_match(s, m, pat)) {
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources is not in the format of cpux or gpux";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
int32_t gpu_index = std::stoi(value.substr(3));
if (s.compare(0, 3, "gpu") == 0) {
int32_t gpu_index = std::stoi(s.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid gpu device: " + value +
std::string msg = "Invalid search resource: " + value +
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
#endif
return Status::OK();
}
......@@ -724,38 +733,20 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
return Status(SERVER_INVALID_ARGUMENT, msg);
}
bool cpu_found = false, gpu_found = false;
for (auto& device : value) {
if (device == "cpu") {
cpu_found = true;
continue;
}
if (CheckGpuDevice(device).ok()) {
gpu_found = true;
} else {
std::string msg = "Invalid search resource: " + device +
". Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
for (auto& resource : value) {
auto status = CheckResource(resource);
if (!status.ok()) {
return Status(SERVER_INVALID_ARGUMENT, status.message());
}
if (cpu_found && !gpu_found) {
std::string msg =
"Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
// if (value == "cpu") {
// return Status::OK();
// }
if (!CheckGpuDevice(value).ok()) {
std::string msg = "Invalid index build device: " + value +
". Possible reason: resource_config.index_build_device does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
auto status = CheckResource(value);
if (!status.ok()) {
return Status(SERVER_INVALID_ARGUMENT, status.message());
}
return Status::OK();
}
......@@ -796,6 +787,22 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key
return value;
}
std::string
Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim,
const std::string& default_value) {
std::string value;
if (!GetConfigValueInMem(parent_key, child_key, value).ok()) {
std::vector<std::string> sequence = GetConfigNode(parent_key).GetSequence(child_key);
if (sequence.empty()) {
value = default_value;
} else {
server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value);
}
SetConfigValueInMem(parent_key, child_key, value);
}
return value;
}
Status
Config::GetServerConfigAddress(std::string& value) {
value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
......@@ -1019,8 +1026,10 @@ Config::GetResourceConfigMode(std::string& value) {
Status
Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
std::string str =
GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES,
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value);
return CheckResourceConfigSearchResources(value);
}
......@@ -1033,10 +1042,10 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
return s;
}
if (str != "cpu") {
value = std::stoi(str.substr(3));
if (str == "cpu") {
value = CPU_DEVICE_ID;
} else {
value = -1;
value = std::stoi(str.substr(3));
}
return Status::OK();
......@@ -1163,7 +1172,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);
return Status::OK();
}
......@@ -1174,7 +1183,7 @@ Config::SetMetricConfigCollector(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);
return Status::OK();
}
......@@ -1185,7 +1194,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);
return Status::OK();
}
......@@ -1197,7 +1206,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
return Status::OK();
}
......@@ -1208,7 +1217,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
return Status::OK();
}
......@@ -1219,7 +1228,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
return Status::OK();
}
......@@ -1230,7 +1239,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
return Status::OK();
}
......@@ -1241,7 +1250,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);
return Status::OK();
}
......@@ -1253,7 +1262,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
return Status::OK();
}
......@@ -1264,7 +1273,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);
return Status::OK();
}
......@@ -1275,7 +1284,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
return Status::OK();
}
......@@ -1287,7 +1296,21 @@ Config::SetResourceConfigMode(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);
return Status::OK();
}
Status
Config::SetResourceConfigSearchResources(const std::string& value) {
std::vector<std::string> res_vec;
server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec);
Status s = CheckResourceConfigSearchResources(res_vec);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value);
return Status::OK();
}
......@@ -1298,7 +1321,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
return Status::OK();
}
......
......@@ -92,12 +92,19 @@ static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu";
#else
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";
#endif
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu";
#else
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
#endif
const int32_t CPU_DEVICE_ID = -1;
class Config {
public:
......@@ -185,6 +192,9 @@ class Config {
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
std::string
GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
const std::string& default_value = "");
public:
/* server config */
......@@ -306,6 +316,8 @@ class Config {
Status
SetResourceConfigMode(const std::string& value);
Status
SetResourceConfigSearchResources(const std::string& value);
Status
SetResourceConfigIndexBuildDevice(const std::string& value);
private:
......
......@@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou
}
}
Status
void
StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter,
std::vector<std::string>& result) {
if (str.empty()) {
return Status::OK();
return;
}
size_t last = 0;
size_t index = str.find_first_of(delimeter, last);
while (index != std::string::npos) {
result.emplace_back(str.substr(last, index - last));
last = index + 1;
index = str.find_first_of(delimeter, last);
size_t prev = 0, pos = 0;
while (true) {
pos = str.find_first_of(delimeter, prev);
if (pos == std::string::npos) {
result.emplace_back(str.substr(prev));
break;
} else {
result.emplace_back(str.substr(prev, pos - prev));
prev = pos + 1;
}
if (index - last > 0) {
std::string temp = str.substr(last);
result.emplace_back(temp);
}
}
return Status::OK();
void
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter,
std::string& result) {
if (strs.empty()) {
result = "";
return;
}
result = strs[0];
for (size_t i = 1; i < strs.size(); i++) {
result = result + delimeter + strs[i];
}
}
Status
StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,
std::vector<std::string>& result) {
if (quote.empty()) {
return SplitStringByDelimeter(str, delimeter, result);
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
size_t last = 0;
size_t index = str.find_first_of(quote, last);
if (index == std::string::npos) {
return SplitStringByDelimeter(str, delimeter, result);
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
std::string process_str = str;
......@@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
}
if (!process_str.empty()) {
return SplitStringByDelimeter(process_str, delimeter, result);
SplitStringByDelimeter(process_str, delimeter, result);
}
return Status::OK();
......
......@@ -43,9 +43,12 @@ class StringHelpFunctions {
// ,b, | b |
// ,, | |
// a a
static Status
static void
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
static void
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
// assume the table has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
......
......@@ -77,6 +77,7 @@ set(helper_files
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc
)
......
......@@ -22,6 +22,7 @@
#include "server/Config.h"
#include "server/utils.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace {
......@@ -98,6 +99,326 @@ TEST_F(ConfigTest, CONFIG_TEST) {
ASSERT_TRUE(seqs.empty());
}
TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s;
std::string str_val;
int32_t int32_val;
int64_t int64_val;
float float_val;
bool bool_val;
/* server config */
std::string server_addr = "192.168.1.155";
s = config.SetServerConfigAddress(server_addr);
ASSERT_TRUE(s.ok());
s = config.GetServerConfigAddress(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == server_addr);
std::string server_port = "12345";
s = config.SetServerConfigPort(server_port);
ASSERT_TRUE(s.ok());
s = config.GetServerConfigPort(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == server_port);
std::string server_mode = "cluster_readonly";
s = config.SetServerConfigDeployMode(server_mode);
ASSERT_TRUE(s.ok());
s = config.GetServerConfigDeployMode(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == server_mode);
std::string server_time_zone = "UTC+6";
s = config.SetServerConfigTimeZone(server_time_zone);
ASSERT_TRUE(s.ok());
s = config.GetServerConfigTimeZone(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == server_time_zone);
/* db config */
std::string db_primary_path = "/home/zilliz";
s = config.SetDBConfigPrimaryPath(db_primary_path);
ASSERT_TRUE(s.ok());
s = config.GetDBConfigPrimaryPath(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == db_primary_path);
std::string db_secondary_path = "/home/zilliz";
s = config.SetDBConfigSecondaryPath(db_secondary_path);
ASSERT_TRUE(s.ok());
s = config.GetDBConfigSecondaryPath(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == db_secondary_path);
std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus";
s = config.SetDBConfigBackendUrl(db_backend_url);
ASSERT_TRUE(s.ok());
s = config.GetDBConfigBackendUrl(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == db_backend_url);
int32_t db_archive_disk_threshold = 100;
s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold));
ASSERT_TRUE(s.ok());
s = config.GetDBConfigArchiveDiskThreshold(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_archive_disk_threshold);
int32_t db_archive_days_threshold = 365;
s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold));
ASSERT_TRUE(s.ok());
s = config.GetDBConfigArchiveDaysThreshold(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_archive_days_threshold);
int32_t db_insert_buffer_size = 2;
s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size));
ASSERT_TRUE(s.ok());
s = config.GetDBConfigInsertBufferSize(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_insert_buffer_size);
/* metric config */
bool metric_enable_monitor = false;
s = config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor));
ASSERT_TRUE(s.ok());
s = config.GetMetricConfigEnableMonitor(bool_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(bool_val == metric_enable_monitor);
std::string metric_collector = "prometheus";
s = config.SetMetricConfigCollector(metric_collector);
ASSERT_TRUE(s.ok());
s = config.GetMetricConfigCollector(str_val);
ASSERT_TRUE(str_val == metric_collector);
std::string metric_prometheus_port = "2222";
s = config.SetMetricConfigPrometheusPort(metric_prometheus_port);
ASSERT_TRUE(s.ok());
s = config.GetMetricConfigPrometheusPort(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == metric_prometheus_port);
/* cache config */
int64_t cache_cpu_cache_capacity = 5;
s = config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity));
ASSERT_TRUE(s.ok());
s = config.GetCacheConfigCpuCacheCapacity(int64_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int64_val == cache_cpu_cache_capacity);
float cache_cpu_cache_threshold = 0.1;
s = config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold));
ASSERT_TRUE(s.ok());
s = config.GetCacheConfigCpuCacheThreshold(float_val);
ASSERT_TRUE(float_val == cache_cpu_cache_threshold);
int64_t cache_gpu_cache_capacity = 1;
s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity));
ASSERT_TRUE(s.ok());
s = config.GetCacheConfigGpuCacheCapacity(int64_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int64_val == cache_gpu_cache_capacity);
float cache_gpu_cache_threshold = 0.2;
s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold));
ASSERT_TRUE(s.ok());
s = config.GetCacheConfigGpuCacheThreshold(float_val);
ASSERT_TRUE(float_val == cache_gpu_cache_threshold);
bool cache_insert_data = true;
s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data));
ASSERT_TRUE(s.ok());
s = config.GetCacheConfigCacheInsertData(bool_val);
ASSERT_TRUE(bool_val == cache_insert_data);
/* engine config */
int32_t engine_use_blas_threshold = 50;
s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold));
ASSERT_TRUE(s.ok());
s = config.GetEngineConfigUseBlasThreshold(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_use_blas_threshold);
int32_t engine_omp_thread_num = 8;
s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num));
ASSERT_TRUE(s.ok());
s = config.GetEngineConfigOmpThreadNum(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_omp_thread_num);
int32_t engine_gpu_search_threshold = 800;
s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));
ASSERT_TRUE(s.ok());
s = config.GetEngineConfigGpuSearchThreshold(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_gpu_search_threshold);
/* resource config */
std::string resource_mode = "simple";
s = config.SetResourceConfigMode(resource_mode);
ASSERT_TRUE(s.ok());
s = config.GetResourceConfigMode(str_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == resource_mode);
#ifdef MILVUS_CPU_VERSION
std::vector<std::string> search_resources = {"cpu"};
#else
std::vector<std::string> search_resources = {"cpu", "gpu0"};
#endif
std::vector<std::string> res_vec;
std::string res_str;
milvus::server::StringHelpFunctions::MergeStringWithDelimeter(
search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str);
s = config.SetResourceConfigSearchResources(res_str);
ASSERT_TRUE(s.ok());
s = config.GetResourceConfigSearchResources(res_vec);
ASSERT_TRUE(s.ok());
for (size_t i = 0; i < search_resources.size(); i++) {
ASSERT_TRUE(search_resources[i] == res_vec[i]);
}
#ifdef MILVUS_CPU_VERSION
int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID;
s = config.SetResourceConfigIndexBuildDevice("cpu");
#else
int32_t resource_index_build_device = 0;
s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device));
#endif
ASSERT_TRUE(s.ok());
s = config.GetResourceConfigIndexBuildDevice(int32_val);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == resource_index_build_device);
}
TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s;
s = config.LoadConfigFile("");
ASSERT_FALSE(s.ok());
s = config.LoadConfigFile(config_path + INVALID_CONFIG_FILE);
ASSERT_FALSE(s.ok());
s = config.LoadConfigFile(config_path + "dummy.yaml");
ASSERT_FALSE(s.ok());
/* server config */
s = config.SetServerConfigAddress("0.0.0");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigAddress("0.0.0.256");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigPort("a");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigPort("99999");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigDeployMode("cluster");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigTimeZone("GM");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigTimeZone("GMT8");
ASSERT_FALSE(s.ok());
s = config.SetServerConfigTimeZone("UTCA");
ASSERT_FALSE(s.ok());
/* db config */
s = config.SetDBConfigPrimaryPath("");
ASSERT_FALSE(s.ok());
// s = config.SetDBConfigSecondaryPath("");
// ASSERT_FALSE(s.ok());
s = config.SetDBConfigBackendUrl("http://www.google.com");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigBackendUrl("sqlite://:@:");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigArchiveDiskThreshold("0x10");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigArchiveDaysThreshold("0x10");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigInsertBufferSize("a");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigInsertBufferSize("0");
ASSERT_FALSE(s.ok());
s = config.SetDBConfigInsertBufferSize("2048");
ASSERT_FALSE(s.ok());
/* metric config */
s = config.SetMetricConfigEnableMonitor("Y");
ASSERT_FALSE(s.ok());
s = config.SetMetricConfigCollector("zilliz");
ASSERT_FALSE(s.ok());
s = config.SetMetricConfigPrometheusPort("0xff");
ASSERT_FALSE(s.ok());
/* cache config */
s = config.SetCacheConfigCpuCacheCapacity("a");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigCpuCacheCapacity("0");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigCpuCacheCapacity("2048");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigCpuCacheThreshold("a");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigCpuCacheThreshold("1.0");
ASSERT_FALSE(s.ok());
#ifdef MILVUS_GPU_VERSION
s = config.SetCacheConfigGpuCacheCapacity("a");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigGpuCacheCapacity("128");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigGpuCacheThreshold("a");
ASSERT_FALSE(s.ok());
s = config.SetCacheConfigGpuCacheThreshold("1.0");
ASSERT_FALSE(s.ok());
#endif
s = config.SetCacheConfigCacheInsertData("N");
ASSERT_FALSE(s.ok());
/* engine config */
s = config.SetEngineConfigUseBlasThreshold("0xff");
ASSERT_FALSE(s.ok());
s = config.SetEngineConfigOmpThreadNum("a");
ASSERT_FALSE(s.ok());
s = config.SetEngineConfigOmpThreadNum("10000");
ASSERT_FALSE(s.ok());
s = config.SetEngineConfigGpuSearchThreshold("-1");
ASSERT_FALSE(s.ok());
/* resource config */
s = config.SetResourceConfigMode("default");
ASSERT_FALSE(s.ok());
s = config.SetResourceConfigSearchResources("gpu10");
ASSERT_FALSE(s.ok());
s = config.SetResourceConfigIndexBuildDevice("gup2");
ASSERT_FALSE(s.ok());
s = config.SetResourceConfigIndexBuildDevice("gpu16");
ASSERT_FALSE(s.ok());
}
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
......
......@@ -117,12 +117,11 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
str = "a,b,c";
std::vector<std::string> result;
auto status = milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
ASSERT_TRUE(status.ok());
milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
ASSERT_EQ(result.size(), 3UL);
result.clear();
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
auto status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册