未验证 提交 68457e33 编写于 作者: J Jin Hai 提交者: GitHub

Merge pull request #430 from cydrain/caiyd_use_all_int64

#306 use int64 for config integer
...@@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA. ...@@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA.
- \#346 - Support build index with multiple gpu - \#346 - Support build index with multiple gpu
## Improvement ## Improvement
- \#275 - Rename C++ SDK IndexType - \#255 - Add ivfsq8 test report detailed version
- \#284 - Change C++ SDK to shared library
- \#260 - C++ SDK README - \#260 - C++ SDK README
- \#266 - Rpc request source code refactor - \#266 - Rpc request source code refactor
- \#314 - add Find FAISS in CMake - \#275 - Rename C++ SDK IndexType
- \#284 - Change C++ SDK to shared library
- \#306 - Use int64 for all config integer
- \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
- \#314 - add Find FAISS in CMake
- \#322 - Add option to enable / disable prometheus - \#322 - Add option to enable / disable prometheus
- \#358 - Add more information in build.sh and install.md - \#358 - Add more information in build.sh and install.md
- \#255 - Add ivfsq8 test report detailed version
- \#404 - Add virtual method Init() in Pass abstract class - \#404 - Add virtual method Init() in Pass abstract class
- \#409 - Add a Fallback pass in optimizer - \#409 - Add a Fallback pass in optimizer
......
...@@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; ...@@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days"; static const char* ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf { struct ArchiveConf {
using CriteriaT = std::map<std::string, int>; using CriteriaT = std::map<std::string, int64_t>;
explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string());
......
...@@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const { ...@@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const {
const std::string key = location_ + ".quantizer"; const std::string key = location_ + ".quantizer";
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
std::vector<int32_t> gpus; std::vector<int64_t> gpus;
Status s = config.GetGpuResourceConfigSearchResources(gpus); Status s = config.GetGpuResourceConfigSearchResources(gpus);
if (!s.ok()) { if (!s.ok()) {
ENGINE_LOG_ERROR << s.message(); ENGINE_LOG_ERROR << s.message();
...@@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { ...@@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status Status
ExecutionEngineImpl::Init() { ExecutionEngineImpl::Init() {
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
std::vector<int32_t> gpu_ids; std::vector<int64_t> gpu_ids;
Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids);
for (auto id : gpu_ids) { for (auto id : gpu_ids) {
if (gpu_num_ == id) { if (gpu_num_ == id) {
......
...@@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { ...@@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine {
int64_t dim_; int64_t dim_;
std::string location_; std::string location_;
int32_t nlist_ = 0; int64_t nlist_ = 0;
int32_t gpu_num_ = 0; int64_t gpu_num_ = 0;
}; };
} // namespace engine } // namespace engine
......
...@@ -55,9 +55,9 @@ load_simple_config() { ...@@ -55,9 +55,9 @@ load_simple_config() {
// get resources // get resources
#ifdef MILVUS_GPU_VERSION #ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
std::vector<int32_t> gpu_ids; std::vector<int64_t> gpu_ids;
config.GetGpuResourceConfigSearchResources(gpu_ids); config.GetGpuResourceConfigSearchResources(gpu_ids);
std::vector<int32_t> build_gpu_ids; std::vector<int64_t> build_gpu_ids;
config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids);
auto pcie = Connection("pcie", 12000); auto pcie = Connection("pcie", 12000);
......
...@@ -26,7 +26,7 @@ namespace scheduler { ...@@ -26,7 +26,7 @@ namespace scheduler {
void void
BuildIndexPass::Init() { BuildIndexPass::Init() {
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
std::vector<int32_t> build_resources; std::vector<int64_t> build_resources;
Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources);
if (!s.ok()) { if (!s.ok()) {
throw; throw;
......
...@@ -45,9 +45,9 @@ class FaissFlatPass : public Pass { ...@@ -45,9 +45,9 @@ class FaissFlatPass : public Pass {
Run(const TaskPtr& task) override; Run(const TaskPtr& task) override;
private: private:
int32_t threshold_ = std::numeric_limits<int32_t>::max(); int64_t threshold_ = std::numeric_limits<int64_t>::max();
int64_t count_ = 0; int64_t count_ = 0;
std::vector<int32_t> gpus; std::vector<int64_t> gpus;
}; };
using FaissFlatPassPtr = std::shared_ptr<FaissFlatPass>; using FaissFlatPassPtr = std::shared_ptr<FaissFlatPass>;
......
...@@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass { ...@@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass {
Run(const TaskPtr& task) override; Run(const TaskPtr& task) override;
private: private:
int32_t threshold_ = std::numeric_limits<int32_t>::max(); int64_t threshold_ = std::numeric_limits<int64_t>::max();
int64_t count_ = 0; int64_t count_ = 0;
std::vector<int32_t> gpus; std::vector<int64_t> gpus;
}; };
using FaissIVFFlatPassPtr = std::shared_ptr<FaissIVFFlatPass>; using FaissIVFFlatPassPtr = std::shared_ptr<FaissIVFFlatPass>;
......
...@@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() { ...@@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() {
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
if (!s.ok()) { if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max(); threshold_ = std::numeric_limits<int64_t>::max();
} }
s = config.GetGpuResourceConfigSearchResources(gpus); s = config.GetGpuResourceConfigSearchResources(gpus);
} }
......
...@@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass { ...@@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass {
Run(const TaskPtr& task) override; Run(const TaskPtr& task) override;
private: private:
int32_t threshold_ = std::numeric_limits<int32_t>::max(); int64_t threshold_ = std::numeric_limits<int64_t>::max();
int64_t count_ = 0; int64_t count_ = 0;
std::vector<int32_t> gpus; std::vector<int64_t> gpus;
}; };
using FaissIVFSQ8HPassPtr = std::shared_ptr<FaissIVFSQ8HPass>; using FaissIVFSQ8HPassPtr = std::shared_ptr<FaissIVFSQ8HPass>;
......
...@@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass { ...@@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass {
Run(const TaskPtr& task) override; Run(const TaskPtr& task) override;
private: private:
int32_t threshold_ = std::numeric_limits<int32_t>::max(); int64_t threshold_ = std::numeric_limits<int64_t>::max();
int64_t count_ = 0; int64_t count_ = 0;
std::vector<int32_t> gpus; std::vector<int64_t> gpus;
}; };
using FaissIVFSQ8PassPtr = std::shared_ptr<FaissIVFSQ8Pass>; using FaissIVFSQ8PassPtr = std::shared_ptr<FaissIVFSQ8Pass>;
......
...@@ -113,19 +113,19 @@ Config::ValidateConfig() { ...@@ -113,19 +113,19 @@ Config::ValidateConfig() {
return s; return s;
} }
int32_t db_archive_disk_threshold; int64_t db_archive_disk_threshold;
s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
int32_t db_archive_days_threshold; int64_t db_archive_days_threshold;
s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
int32_t db_insert_buffer_size; int64_t db_insert_buffer_size;
s = GetDBConfigInsertBufferSize(db_insert_buffer_size); s = GetDBConfigInsertBufferSize(db_insert_buffer_size);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
...@@ -170,19 +170,19 @@ Config::ValidateConfig() { ...@@ -170,19 +170,19 @@ Config::ValidateConfig() {
} }
/* engine config */ /* engine config */
int32_t engine_use_blas_threshold; int64_t engine_use_blas_threshold;
s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
int32_t engine_omp_thread_num; int64_t engine_omp_thread_num;
s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); s = GetEngineConfigOmpThreadNum(engine_omp_thread_num);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
int32_t engine_gpu_search_threshold; int64_t engine_gpu_search_threshold;
s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
...@@ -208,13 +208,13 @@ Config::ValidateConfig() { ...@@ -208,13 +208,13 @@ Config::ValidateConfig() {
return s; return s;
} }
std::vector<int32_t> search_resources; std::vector<int64_t> search_resources;
s = GetGpuResourceConfigSearchResources(search_resources); s = GetGpuResourceConfigSearchResources(search_resources);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
std::vector<int32_t> index_build_resources; std::vector<int64_t> index_build_resources;
s = GetGpuResourceConfigBuildIndexResources(index_build_resources); s = GetGpuResourceConfigBuildIndexResources(index_build_resources);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
...@@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ...@@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
". Possible reason: db_config.insert_buffer_size is not a positive integer."; ". Possible reason: db_config.insert_buffer_size is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg); return Status(SERVER_INVALID_ARGUMENT, msg);
} else { } else {
int64_t buffer_size = std::stoi(value) * GB; int64_t buffer_size = std::stoll(value) * GB;
if (buffer_size <= 0) { if (buffer_size <= 0) {
std::string msg = "Invalid insert buffer size: " + value + std::string msg = "Invalid insert buffer size: " + value +
". Possible reason: db_config.insert_buffer_size is not a positive integer."; ". Possible reason: db_config.insert_buffer_size is not a positive integer.";
...@@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ...@@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg); return Status(SERVER_INVALID_ARGUMENT, msg);
} else { } else {
int64_t cpu_cache_capacity = std::stoi(value) * GB; int64_t cpu_cache_capacity = std::stoll(value) * GB;
if (cpu_cache_capacity <= 0) { if (cpu_cache_capacity <= 0) {
std::string msg = "Invalid cpu cache capacity: " + value + std::string msg = "Invalid cpu cache capacity: " + value +
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
...@@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ...@@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl;
} }
int32_t buffer_value; int64_t buffer_value;
Status s = GetDBConfigInsertBufferSize(buffer_value); Status s = GetDBConfigInsertBufferSize(buffer_value);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
...@@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { ...@@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
return Status(SERVER_INVALID_ARGUMENT, msg); return Status(SERVER_INVALID_ARGUMENT, msg);
} }
int32_t omp_thread = std::stoi(value); int64_t omp_thread = std::stoll(value);
uint32_t sys_thread_cnt = 8; int64_t sys_thread_cnt = 8;
CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); CommonUtil::GetSystemAvailableThreads(sys_thread_cnt);
if (omp_thread > static_cast<int32_t>(sys_thread_cnt)) { if (omp_thread > sys_thread_cnt) {
std::string msg = "Invalid omp thread num: " + value + std::string msg = "Invalid omp thread num: " + value +
". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores.";
return Status(SERVER_INVALID_ARGUMENT, msg); return Status(SERVER_INVALID_ARGUMENT, msg);
...@@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { ...@@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg); return Status(SERVER_INVALID_ARGUMENT, msg);
} else { } else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB; int64_t gpu_cache_capacity = std::stoll(value) * GB;
std::vector<int32_t> gpu_ids; std::vector<int64_t> gpu_ids;
Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
for (int32_t gpu_id : gpu_ids) { for (int64_t gpu_id : gpu_ids) {
size_t gpu_memory; size_t gpu_memory;
if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id);
...@@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { ...@@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) {
} }
Status Status
Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Config::GetDBConfigArchiveDiskThreshold(int64_t& value) {
std::string str = std::string str =
GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT);
Status s = CheckDBConfigArchiveDiskThreshold(str); Status s = CheckDBConfigArchiveDiskThreshold(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
Status Status
Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Config::GetDBConfigArchiveDaysThreshold(int64_t& value) {
std::string str = std::string str =
GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT);
Status s = CheckDBConfigArchiveDaysThreshold(str); Status s = CheckDBConfigArchiveDaysThreshold(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
Status Status
Config::GetDBConfigInsertBufferSize(int32_t& value) { Config::GetDBConfigInsertBufferSize(int64_t& value) {
std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT);
Status s = CheckDBConfigInsertBufferSize(str); Status s = CheckDBConfigInsertBufferSize(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
...@@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { ...@@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) {
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
...@@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) { ...@@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) {
} }
Status Status
Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Config::GetEngineConfigUseBlasThreshold(int64_t& value) {
std::string str = std::string str =
GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT);
Status s = CheckEngineConfigUseBlasThreshold(str); Status s = CheckEngineConfigUseBlasThreshold(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
Status Status
Config::GetEngineConfigOmpThreadNum(int32_t& value) { Config::GetEngineConfigOmpThreadNum(int64_t& value) {
std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT);
Status s = CheckEngineConfigOmpThreadNum(str); Status s = CheckEngineConfigOmpThreadNum(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
Status Status
Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { Config::GetEngineConfigGpuSearchThreshold(int64_t& value) {
std::string str = std::string str =
GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT);
Status s = CheckEngineConfigGpuSearchThreshold(str); Status s = CheckEngineConfigGpuSearchThreshold(str);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
...@@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { ...@@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) {
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
value = std::stoi(str); value = std::stoll(str);
return Status::OK(); return Status::OK();
} }
...@@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { ...@@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) {
} }
Status Status
Config::GetGpuResourceConfigSearchResources(std::vector<int32_t>& value) { Config::GetGpuResourceConfigSearchResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false; bool gpu_resource_enable = false;
Status s = GetGpuResourceConfigEnable(gpu_resource_enable); Status s = GetGpuResourceConfigEnable(gpu_resource_enable);
if (!s.ok()) { if (!s.ok()) {
...@@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector<int32_t>& value) { ...@@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector<int32_t>& value) {
return s; return s;
} }
for (std::string& res : res_vec) { for (std::string& res : res_vec) {
value.push_back(std::stoi(res.substr(3))); value.push_back(std::stoll(res.substr(3)));
} }
return Status::OK(); return Status::OK();
} }
Status Status
Config::GetGpuResourceConfigBuildIndexResources(std::vector<int32_t>& value) { Config::GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value) {
bool gpu_resource_enable = false; bool gpu_resource_enable = false;
Status s = GetGpuResourceConfigEnable(gpu_resource_enable); Status s = GetGpuResourceConfigEnable(gpu_resource_enable);
if (!s.ok()) { if (!s.ok()) {
...@@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector<int32_t>& value) { ...@@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector<int32_t>& value) {
return s; return s;
} }
for (std::string& res : res_vec) { for (std::string& res : res_vec) {
value.push_back(std::stoi(res.substr(3))); value.push_back(std::stoll(res.substr(3)));
} }
return Status::OK(); return Status::OK();
} }
......
...@@ -210,11 +210,11 @@ class Config { ...@@ -210,11 +210,11 @@ class Config {
Status Status
GetDBConfigBackendUrl(std::string& value); GetDBConfigBackendUrl(std::string& value);
Status Status
GetDBConfigArchiveDiskThreshold(int32_t& value); GetDBConfigArchiveDiskThreshold(int64_t& value);
Status Status
GetDBConfigArchiveDaysThreshold(int32_t& value); GetDBConfigArchiveDaysThreshold(int64_t& value);
Status Status
GetDBConfigInsertBufferSize(int32_t& value); GetDBConfigInsertBufferSize(int64_t& value);
Status Status
GetDBConfigPreloadTable(std::string& value); GetDBConfigPreloadTable(std::string& value);
...@@ -236,11 +236,11 @@ class Config { ...@@ -236,11 +236,11 @@ class Config {
/* engine config */ /* engine config */
Status Status
GetEngineConfigUseBlasThreshold(int32_t& value); GetEngineConfigUseBlasThreshold(int64_t& value);
Status Status
GetEngineConfigOmpThreadNum(int32_t& value); GetEngineConfigOmpThreadNum(int64_t& value);
Status Status
GetEngineConfigGpuSearchThreshold(int32_t& value); GetEngineConfigGpuSearchThreshold(int64_t& value);
/* gpu resource config */ /* gpu resource config */
Status Status
...@@ -250,9 +250,9 @@ class Config { ...@@ -250,9 +250,9 @@ class Config {
Status Status
GetGpuResourceConfigCacheThreshold(float& value); GetGpuResourceConfigCacheThreshold(float& value);
Status Status
GetGpuResourceConfigSearchResources(std::vector<int32_t>& value); GetGpuResourceConfigSearchResources(std::vector<int64_t>& value);
Status Status
GetGpuResourceConfigBuildIndexResources(std::vector<int32_t>& value); GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value);
public: public:
/* server config */ /* server config */
......
...@@ -89,7 +89,7 @@ DBWrapper::StartService() { ...@@ -89,7 +89,7 @@ DBWrapper::StartService() {
} }
// engine config // engine config
int32_t omp_thread; int64_t omp_thread;
s = config.GetEngineConfigOmpThreadNum(omp_thread); s = config.GetEngineConfigOmpThreadNum(omp_thread);
if (!s.ok()) { if (!s.ok()) {
std::cerr << s.ToString() << std::endl; std::cerr << s.ToString() << std::endl;
...@@ -100,7 +100,7 @@ DBWrapper::StartService() { ...@@ -100,7 +100,7 @@ DBWrapper::StartService() {
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread;
} else { } else {
uint32_t sys_thread_cnt = 8; int64_t sys_thread_cnt = 8;
if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = static_cast<int32_t>(ceil(sys_thread_cnt * 0.5)); omp_thread = static_cast<int32_t>(ceil(sys_thread_cnt * 0.5));
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
...@@ -108,7 +108,7 @@ DBWrapper::StartService() { ...@@ -108,7 +108,7 @@ DBWrapper::StartService() {
} }
// init faiss global variable // init faiss global variable
int32_t use_blas_threshold; int64_t use_blas_threshold;
s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold);
if (!s.ok()) { if (!s.ok()) {
std::cerr << s.ToString() << std::endl; std::cerr << s.ToString() << std::endl;
...@@ -119,7 +119,7 @@ DBWrapper::StartService() { ...@@ -119,7 +119,7 @@ DBWrapper::StartService() {
// set archive config // set archive config
engine::ArchiveConf::CriteriaT criterial; engine::ArchiveConf::CriteriaT criterial;
int32_t disk, days; int64_t disk, days;
s = config.GetDBConfigArchiveDiskThreshold(disk); s = config.GetDBConfigArchiveDiskThreshold(disk);
if (!s.ok()) { if (!s.ok()) {
std::cerr << s.ToString() << std::endl; std::cerr << s.ToString() << std::endl;
......
...@@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { ...@@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) {
} }
bool bool
CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency(); // threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU; thread_count *= THREAD_MULTIPLY_CPU;
......
...@@ -30,7 +30,7 @@ class CommonUtil { ...@@ -30,7 +30,7 @@ class CommonUtil {
static bool static bool
GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem);
static bool static bool
GetSystemAvailableThreads(uint32_t& thread_count); GetSystemAvailableThreads(int64_t& thread_count);
static bool static bool
IsFileExist(const std::string& path); IsFileExist(const std::string& path);
......
...@@ -48,7 +48,7 @@ KnowhereResource::Initialize() { ...@@ -48,7 +48,7 @@ KnowhereResource::Initialize() {
// get build index gpu resource // get build index gpu resource
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
std::vector<int32_t> build_index_gpus; std::vector<int64_t> build_index_gpus;
s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus);
if (!s.ok()) if (!s.ok())
return s; return s;
...@@ -58,7 +58,7 @@ KnowhereResource::Initialize() { ...@@ -58,7 +58,7 @@ KnowhereResource::Initialize() {
} }
// get search gpu resource // get search gpu resource
std::vector<int32_t> search_gpus; std::vector<int64_t> search_gpus;
s = config.GetGpuResourceConfigSearchResources(search_gpus); s = config.GetGpuResourceConfigSearchResources(search_gpus);
if (!s.ok()) if (!s.ok())
return s; return s;
......
...@@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s; milvus::Status s;
std::string str_val; std::string str_val;
int32_t int32_val;
int64_t int64_val; int64_t int64_val;
float float_val; float float_val;
bool bool_val; bool bool_val;
...@@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(str_val == db_backend_url); ASSERT_TRUE(str_val == db_backend_url);
int32_t db_archive_disk_threshold = 100; int64_t db_archive_disk_threshold = 100;
s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetDBConfigArchiveDiskThreshold(int32_val); s = config.GetDBConfigArchiveDiskThreshold(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_archive_disk_threshold); ASSERT_TRUE(int64_val == db_archive_disk_threshold);
int32_t db_archive_days_threshold = 365; int64_t db_archive_days_threshold = 365;
s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetDBConfigArchiveDaysThreshold(int32_val); s = config.GetDBConfigArchiveDaysThreshold(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_archive_days_threshold); ASSERT_TRUE(int64_val == db_archive_days_threshold);
int32_t db_insert_buffer_size = 2; int64_t db_insert_buffer_size = 2;
s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetDBConfigInsertBufferSize(int32_val); s = config.GetDBConfigInsertBufferSize(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == db_insert_buffer_size); ASSERT_TRUE(int64_val == db_insert_buffer_size);
/* metric config */ /* metric config */
bool metric_enable_monitor = false; bool metric_enable_monitor = false;
...@@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(bool_val == cache_insert_data); ASSERT_TRUE(bool_val == cache_insert_data);
/* engine config */ /* engine config */
int32_t engine_use_blas_threshold = 50; int64_t engine_use_blas_threshold = 50;
s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetEngineConfigUseBlasThreshold(int32_val); s = config.GetEngineConfigUseBlasThreshold(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_use_blas_threshold); ASSERT_TRUE(int64_val == engine_use_blas_threshold);
int32_t engine_omp_thread_num = 8; int64_t engine_omp_thread_num = 8;
s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetEngineConfigOmpThreadNum(int32_val); s = config.GetEngineConfigOmpThreadNum(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_omp_thread_num); ASSERT_TRUE(int64_val == engine_omp_thread_num);
int32_t engine_gpu_search_threshold = 800; int64_t engine_gpu_search_threshold = 800;
s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
s = config.GetEngineConfigGpuSearchThreshold(int32_val); s = config.GetEngineConfigGpuSearchThreshold(int64_val);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
ASSERT_TRUE(int32_val == engine_gpu_search_threshold); ASSERT_TRUE(int64_val == engine_gpu_search_threshold);
/* gpu resource config */ /* gpu resource config */
bool resource_enable_gpu = true; bool resource_enable_gpu = true;
...@@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(float_val == gpu_cache_threshold); ASSERT_TRUE(float_val == gpu_cache_threshold);
std::vector<std::string> search_resources = {"gpu0"}; std::vector<std::string> search_resources = {"gpu0"};
std::vector<int32_t> search_res_vec; std::vector<int64_t> search_res_vec;
std::string search_res_str; std::string search_res_str;
milvus::server::StringHelpFunctions::MergeStringWithDelimeter( milvus::server::StringHelpFunctions::MergeStringWithDelimeter(
search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str);
...@@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
s = config.GetGpuResourceConfigSearchResources(search_res_vec); s = config.GetGpuResourceConfigSearchResources(search_res_vec);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
for (size_t i = 0; i < search_resources.size(); i++) { for (size_t i = 0; i < search_resources.size(); i++) {
ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]);
} }
std::vector<std::string> build_index_resources = {"gpu0"}; std::vector<std::string> build_index_resources = {"gpu0"};
std::vector<int32_t> build_index_res_vec; std::vector<int64_t> build_index_res_vec;
std::string build_index_res_str; std::string build_index_res_str;
milvus::server::StringHelpFunctions::MergeStringWithDelimeter( milvus::server::StringHelpFunctions::MergeStringWithDelimeter(
build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str);
...@@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ...@@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec);
ASSERT_TRUE(s.ok()); ASSERT_TRUE(s.ok());
for (size_t i = 0; i < build_index_resources.size(); i++) { for (size_t i = 0; i < build_index_resources.size(); i++) {
ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]);
} }
#endif #endif
} }
......
...@@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ...@@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) {
ASSERT_GT(total_mem, 0); ASSERT_GT(total_mem, 0);
ASSERT_GT(free_mem, 0); ASSERT_GT(free_mem, 0);
uint32_t thread_cnt = 0; int64_t thread_cnt = 0;
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
ASSERT_GT(thread_cnt, 0); ASSERT_GT(thread_cnt, 0);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册