Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 dd29a72d 编写于 作者: G groot
浏览文件
操作

format server code 


Former-commit-id: 2fb1177818acab220d905b9051e87bf8b636240d
上级 a42ade88
展开全部 隐藏空白更改
内联 并排
Showing with 599 addition and 611 deletion
cpp/src/server/Config.cpp
浏览文件 @ dd29a72d
此差异已折叠。
cpp/src/server/Config.h
浏览文件 @ dd29a72d
......@@ -17,136 +17,137 @@
#pragma once
#include <vector>
#include <string>
#include <mutex>
#include <unordered_map>
#include "yaml-cpp/yaml.h"
#include <yaml-cpp/yaml.h>
#include "utils/Status.h"
#include "config/ConfigNode.h"
namespace zilliz {
namespace milvus {
namespace server {
/* server config */
static const char* CONFIG_SERVER = "server_config";
static const char* CONFIG_SERVER_ADDRESS = "address";
static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1";
static const char* CONFIG_SERVER_PORT = "port";
static const char* CONFIG_SERVER_PORT_DEFAULT = "19530";
static const char* CONFIG_SERVER_MODE = "mode";
static const char* CONFIG_SERVER_MODE_DEFAULT = "single";
static const char* CONFIG_SERVER_TIME_ZONE = "time_zone";
static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
static const char *CONFIG_SERVER = "server_config";
static const char *CONFIG_SERVER_ADDRESS = "address";
static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1";
static const char *CONFIG_SERVER_PORT = "port";
static const char *CONFIG_SERVER_PORT_DEFAULT = "19530";
static const char *CONFIG_SERVER_MODE = "mode";
static const char *CONFIG_SERVER_MODE_DEFAULT = "single";
static const char *CONFIG_SERVER_TIME_ZONE = "time_zone";
static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
/* db config */
static const char* CONFIG_DB = "db_config";
static const char* CONFIG_DB_PATH = "path";
static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus";
static const char* CONFIG_DB_SLAVE_PATH = "slave_path";
static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = "";
static const char* CONFIG_DB_BACKEND_URL = "backend_url";
static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size";
static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
static const char *CONFIG_DB = "db_config";
static const char *CONFIG_DB_PATH = "path";
static const char *CONFIG_DB_PATH_DEFAULT = "/tmp/milvus";
static const char *CONFIG_DB_SLAVE_PATH = "slave_path";
static const char *CONFIG_DB_SLAVE_PATH_DEFAULT = "";
static const char *CONFIG_DB_BACKEND_URL = "backend_url";
static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/";
static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char *CONFIG_DB_BUFFER_SIZE = "buffer_size";
static const char *CONFIG_DB_BUFFER_SIZE_DEFAULT = "4";
static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
/* cache config */
static const char* CONFIG_CACHE = "cache_config";
static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity";
static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16";
static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity";
static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0";
static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold";
static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false";
static const char *CONFIG_CACHE = "cache_config";
static const char *CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity";
static const char *CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16";
static const char *CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity";
static const char *CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0";
static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold";
static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold";
static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data";
static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false";
/* metric config */
static const char* CONFIG_METRIC = "metric_config";
static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup";
static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false";
static const char* CONFIG_METRIC_COLLECTOR = "collector";
static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus";
static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config";
static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port";
static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080";
static const char *CONFIG_METRIC = "metric_config";
static const char *CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup";
static const char *CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false";
static const char *CONFIG_METRIC_COLLECTOR = "collector";
static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus";
static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config";
static const char *CONFIG_METRIC_PROMETHEUS_PORT = "port";
static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080";
/* engine config */
static const char* CONFIG_ENGINE = "engine_config";
static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold";
static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char *CONFIG_ENGINE = "engine_config";
static const char *CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold";
static const char *CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20";
static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
/* resource config */
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_POOL = "pool";
static const char *CONFIG_RESOURCE = "resource_config";
static const char *CONFIG_RESOURCE_MODE = "mode";
static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char *CONFIG_RESOURCE_POOL = "pool";
class Config {
public:
static Config& GetInstance();
Status LoadConfigFile(const std::string& filename);
static Config &GetInstance();
Status LoadConfigFile(const std::string &filename);
void PrintAll();
private:
ConfigNode& GetConfigNode(const std::string& name);
ConfigNode &GetConfigNode(const std::string &name);
Status GetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
std::string& value);
Status GetConfigValueInMem(const std::string &parent_key,
const std::string &child_key,
std::string &value);
void SetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
const std::string& value);
void SetConfigValueInMem(const std::string &parent_key,
const std::string &child_key,
const std::string &value);
void PrintConfigSection(const std::string& config_node_name);
void PrintConfigSection(const std::string &config_node_name);
///////////////////////////////////////////////////////////////////////////
/* server config */
Status CheckServerConfigAddress(const std::string& value);
Status CheckServerConfigPort(const std::string& value);
Status CheckServerConfigMode(const std::string& value);
Status CheckServerConfigTimeZone(const std::string& value);
Status CheckServerConfigAddress(const std::string &value);
Status CheckServerConfigPort(const std::string &value);
Status CheckServerConfigMode(const std::string &value);
Status CheckServerConfigTimeZone(const std::string &value);
/* db config */
Status CheckDBConfigPath(const std::string& value);
Status CheckDBConfigSlavePath(const std::string& value);
Status CheckDBConfigBackendUrl(const std::string& value);
Status CheckDBConfigArchiveDiskThreshold(const std::string& value);
Status CheckDBConfigArchiveDaysThreshold(const std::string& value);
Status CheckDBConfigBufferSize(const std::string& value);
Status CheckDBConfigBuildIndexGPU(const std::string& value);
Status CheckDBConfigPath(const std::string &value);
Status CheckDBConfigSlavePath(const std::string &value);
Status CheckDBConfigBackendUrl(const std::string &value);
Status CheckDBConfigArchiveDiskThreshold(const std::string &value);
Status CheckDBConfigArchiveDaysThreshold(const std::string &value);
Status CheckDBConfigBufferSize(const std::string &value);
Status CheckDBConfigBuildIndexGPU(const std::string &value);
/* metric config */
Status CheckMetricConfigAutoBootup(const std::string& value);
Status CheckMetricConfigCollector(const std::string& value);
Status CheckMetricConfigPrometheusPort(const std::string& value);
Status CheckMetricConfigAutoBootup(const std::string &value);
Status CheckMetricConfigCollector(const std::string &value);
Status CheckMetricConfigPrometheusPort(const std::string &value);
/* cache config */
Status CheckCacheConfigCpuMemCapacity(const std::string& value);
Status CheckCacheConfigCpuMemThreshold(const std::string& value);
Status CheckCacheConfigGpuMemCapacity(const std::string& value);
Status CheckCacheConfigGpuMemThreshold(const std::string& value);
Status CheckCacheConfigCacheInsertData(const std::string& value);
Status CheckCacheConfigCpuMemCapacity(const std::string &value);
Status CheckCacheConfigCpuMemThreshold(const std::string &value);
Status CheckCacheConfigGpuMemCapacity(const std::string &value);
Status CheckCacheConfigGpuMemThreshold(const std::string &value);
Status CheckCacheConfigCacheInsertData(const std::string &value);
/* engine config */
Status CheckEngineConfigBlasThreshold(const std::string& value);
Status CheckEngineConfigOmpThreadNum(const std::string& value);
Status CheckEngineConfigBlasThreshold(const std::string &value);
Status CheckEngineConfigOmpThreadNum(const std::string &value);
/* resource config */
Status CheckResourceConfigMode(const std::string& value);
Status CheckResourceConfigPool(const std::vector<std::string>& value);
Status CheckResourceConfigMode(const std::string &value);
Status CheckResourceConfigPool(const std::vector<std::string> &value);
///////////////////////////////////////////////////////////////////////////
/* server config */
......@@ -185,81 +186,81 @@ class Config {
public:
/* server config */
Status GetServerConfigAddress(std::string& value);
Status GetServerConfigPort(std::string& value);
Status GetServerConfigMode(std::string& value);
Status GetServerConfigTimeZone(std::string& value);
Status GetServerConfigAddress(std::string &value);
Status GetServerConfigPort(std::string &value);
Status GetServerConfigMode(std::string &value);
Status GetServerConfigTimeZone(std::string &value);
/* db config */
Status GetDBConfigPath(std::string& value);
Status GetDBConfigSlavePath(std::string& value);
Status GetDBConfigBackendUrl(std::string& value);
Status GetDBConfigArchiveDiskThreshold(int32_t& value);
Status GetDBConfigArchiveDaysThreshold(int32_t& value);
Status GetDBConfigBufferSize(int32_t& value);
Status GetDBConfigBuildIndexGPU(int32_t& value);
Status GetDBConfigPath(std::string &value);
Status GetDBConfigSlavePath(std::string &value);
Status GetDBConfigBackendUrl(std::string &value);
Status GetDBConfigArchiveDiskThreshold(int32_t &value);
Status GetDBConfigArchiveDaysThreshold(int32_t &value);
Status GetDBConfigBufferSize(int32_t &value);
Status GetDBConfigBuildIndexGPU(int32_t &value);
/* metric config */
Status GetMetricConfigAutoBootup(bool& value);
Status GetMetricConfigCollector(std::string& value);
Status GetMetricConfigPrometheusPort(std::string& value);
Status GetMetricConfigAutoBootup(bool &value);
Status GetMetricConfigCollector(std::string &value);
Status GetMetricConfigPrometheusPort(std::string &value);
/* cache config */
Status GetCacheConfigCpuMemCapacity(int32_t& value);
Status GetCacheConfigCpuMemThreshold(float& value);
Status GetCacheConfigGpuMemCapacity(int32_t& value);
Status GetCacheConfigGpuMemThreshold(float& value);
Status GetCacheConfigCacheInsertData(bool& value);
Status GetCacheConfigCpuMemCapacity(int32_t &value);
Status GetCacheConfigCpuMemThreshold(float &value);
Status GetCacheConfigGpuMemCapacity(int32_t &value);
Status GetCacheConfigGpuMemThreshold(float &value);
Status GetCacheConfigCacheInsertData(bool &value);
/* engine config */
Status GetEngineConfigBlasThreshold(int32_t& value);
Status GetEngineConfigOmpThreadNum(int32_t& value);
Status GetEngineConfigBlasThreshold(int32_t &value);
Status GetEngineConfigOmpThreadNum(int32_t &value);
/* resource config */
Status GetResourceConfigMode(std::string& value);
Status GetResourceConfigPool(std::vector<std::string>& value);
Status GetResourceConfigMode(std::string &value);
Status GetResourceConfigPool(std::vector<std::string> &value);
public:
/* server config */
Status SetServerConfigAddress(const std::string& value);
Status SetServerConfigPort(const std::string& value);
Status SetServerConfigMode(const std::string& value);
Status SetServerConfigTimeZone(const std::string& value);
Status SetServerConfigAddress(const std::string &value);
Status SetServerConfigPort(const std::string &value);
Status SetServerConfigMode(const std::string &value);
Status SetServerConfigTimeZone(const std::string &value);
/* db config */
Status SetDBConfigPath(const std::string& value);
Status SetDBConfigSlavePath(const std::string& value);
Status SetDBConfigBackendUrl(const std::string& value);
Status SetDBConfigArchiveDiskThreshold(const std::string& value);
Status SetDBConfigArchiveDaysThreshold(const std::string& value);
Status SetDBConfigBufferSize(const std::string& value);
Status SetDBConfigBuildIndexGPU(const std::string& value);
Status SetDBConfigPath(const std::string &value);
Status SetDBConfigSlavePath(const std::string &value);
Status SetDBConfigBackendUrl(const std::string &value);
Status SetDBConfigArchiveDiskThreshold(const std::string &value);
Status SetDBConfigArchiveDaysThreshold(const std::string &value);
Status SetDBConfigBufferSize(const std::string &value);
Status SetDBConfigBuildIndexGPU(const std::string &value);
/* metric config */
Status SetMetricConfigAutoBootup(const std::string& value);
Status SetMetricConfigCollector(const std::string& value);
Status SetMetricConfigPrometheusPort(const std::string& value);
Status SetMetricConfigAutoBootup(const std::string &value);
Status SetMetricConfigCollector(const std::string &value);
Status SetMetricConfigPrometheusPort(const std::string &value);
/* cache config */
Status SetCacheConfigCpuMemCapacity(const std::string& value);
Status SetCacheConfigCpuMemThreshold(const std::string& value);
Status SetCacheConfigGpuMemCapacity(const std::string& value);
Status SetCacheConfigGpuMemThreshold(const std::string& value);
Status SetCacheConfigCacheInsertData(const std::string& value);
Status SetCacheConfigCpuMemCapacity(const std::string &value);
Status SetCacheConfigCpuMemThreshold(const std::string &value);
Status SetCacheConfigGpuMemCapacity(const std::string &value);
Status SetCacheConfigGpuMemThreshold(const std::string &value);
Status SetCacheConfigCacheInsertData(const std::string &value);
/* engine config */
Status SetEngineConfigBlasThreshold(const std::string& value);
Status SetEngineConfigOmpThreadNum(const std::string& value);
Status SetEngineConfigBlasThreshold(const std::string &value);
Status SetEngineConfigOmpThreadNum(const std::string &value);
/* resource config */
Status SetResourceConfigMode(const std::string& value);
Status SetResourceConfigMode(const std::string &value);
private:
std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;
std::mutex mutex_;
};
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/DBWrapper.cpp
浏览文件 @ dd29a72d
......@@ -16,13 +16,14 @@
// under the License.
#include "DBWrapper.h"
#include "server/DBWrapper.h"
#include "Config.h"
#include "db/DBFactory.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <string>
#include <omp.h>
#include <faiss/utils.h>
......@@ -31,11 +32,11 @@ namespace milvus {
namespace server {
DBWrapper::DBWrapper() {
}
Status DBWrapper::StartService() {
Config& config = Config::GetInstance();
Status
DBWrapper::StartService() {
Config &config = Config::GetInstance();
Status s;
//db config
engine::DBOptions opt;
......@@ -65,15 +66,13 @@ Status DBWrapper::StartService() {
if (mode == "single") {
opt.mode_ = engine::DBOptions::MODE::SINGLE;
}
else if (mode == "cluster") {
} else if (mode == "cluster") {
opt.mode_ = engine::DBOptions::MODE::CLUSTER;
}
else if (mode == "read_only") {
} else if (mode == "read_only") {
opt.mode_ = engine::DBOptions::MODE::READ_ONLY;
}
else {
std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl;
} else {
std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']"
<< std::endl;
kill(0, SIGUSR1);
}
......@@ -86,8 +85,8 @@ Status DBWrapper::StartService() {
SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread;
} else {
uint32_t sys_thread_cnt = 8;
if(CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = (int32_t)ceil(sys_thread_cnt*0.5);
if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = (int32_t) ceil(sys_thread_cnt * 0.5);
omp_set_num_threads(omp_thread);
}
}
......@@ -116,14 +115,14 @@ Status DBWrapper::StartService() {
//create db root folder
Status status = CommonUtil::CreateDirectory(opt.meta_.path_);
if(!status.ok()) {
if (!status.ok()) {
std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl;
kill(0, SIGUSR1);
}
for(auto& path : opt.meta_.slave_paths_) {
for (auto &path : opt.meta_.slave_paths_) {
status = CommonUtil::CreateDirectory(path);
if(!status.ok()) {
if (!status.ok()) {
std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl;
kill(0, SIGUSR1);
}
......@@ -132,7 +131,7 @@ Status DBWrapper::StartService() {
//create db instance
try {
db_ = engine::DBFactory::Build(opt);
} catch(std::exception& ex) {
} catch (std::exception &ex) {
std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl;
kill(0, SIGUSR1);
}
......@@ -142,14 +141,15 @@ Status DBWrapper::StartService() {
return Status::OK();
}
Status DBWrapper::StopService() {
if(db_) {
Status
DBWrapper::StopService() {
if (db_) {
db_->Stop();
}
return Status::OK();
}
}
}
}
\ No newline at end of file
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/DBWrapper.h
浏览文件 @ dd29a72d
......@@ -27,12 +27,12 @@ namespace milvus {
namespace server {
class DBWrapper {
private:
private:
DBWrapper();
~DBWrapper() = default;
public:
static DBWrapper& GetInstance() {
public:
static DBWrapper &GetInstance() {
static DBWrapper wrapper;
return wrapper;
}
......@@ -48,10 +48,10 @@ public:
return db_;
}
private:
private:
engine::DBPtr db_;
};
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/Server.cpp
浏览文件 @ dd29a72d
......@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include "server/Server.h"
#include <thread>
#include <fcntl.h>
#include <sys/stat.h>
......@@ -24,7 +26,6 @@
#include <unistd.h>
#include <string.h>
#include "Server.h"
#include "server/grpc_impl/GrpcServer.h"
#include "server/Config.h"
#include "utils/Log.h"
......@@ -36,7 +37,6 @@
#include "wrapper/KnowhereResource.h"
#include "DBWrapper.h"
namespace zilliz {
namespace milvus {
namespace server {
......@@ -116,7 +116,7 @@ Server::Daemonize() {
}
// Close all open fd
for (long fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) {
for (int64_t fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) {
close(fd);
}
......@@ -172,9 +172,9 @@ Server::Start() {
time_zone = "CUT";
} else {
int time_bias = std::stoi(time_zone.substr(3, std::string::npos));
if (time_bias == 0)
if (time_bias == 0) {
time_zone = "CUT";
else if (time_bias > 0) {
} else if (time_bias > 0) {
time_zone = "CUT" + std::to_string(-time_bias);
} else {
time_zone = "CUT+" + std::to_string(-time_bias);
......@@ -194,7 +194,6 @@ Server::Start() {
StartService();
std::cout << "Milvus server start successfully." << std::endl;
} catch (std::exception &ex) {
std::cerr << "Milvus server encounter exception: " << ex.what();
}
......@@ -232,10 +231,9 @@ Server::Stop() {
std::cerr << "Milvus server is closed!" << std::endl;
}
ErrorCode
Server::LoadConfig() {
Config& config = Config::GetInstance();
Config &config = Config::GetInstance();
Status s = config.LoadConfigFile(config_filename_);
if (!s.ok()) {
std::cerr << "Failed to load config file: " << config_filename_ << std::endl;
......@@ -260,6 +258,6 @@ Server::StopService() {
engine::KnowhereResource::Finalize();
}
}
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/Server.h
浏览文件 @ dd29a72d
......@@ -22,7 +22,6 @@
#include <cstdint>
#include <string>
namespace zilliz {
namespace milvus {
namespace server {
......@@ -58,6 +57,6 @@ class Server {
std::string log_config_file_;
}; // Server
} // server
} // sql
} // zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @ dd29a72d
......@@ -16,10 +16,12 @@
// under the License.
#include "GrpcRequestHandler.h"
#include "GrpcRequestTask.h"
#include "server/grpc_impl/GrpcRequestHandler.h"
#include "server/grpc_impl/GrpcRequestTask.h"
#include "utils/TimeRecorder.h"
#include <vector>
namespace zilliz {
namespace milvus {
namespace server {
......@@ -29,7 +31,6 @@ namespace grpc {
GrpcRequestHandler::CreateTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableSchema *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = CreateTableTask::Create(request);
GrpcRequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
......@@ -39,7 +40,6 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext *context,
GrpcRequestHandler::HasTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::BoolReply *response) {
bool has_table = false;
BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table);
::milvus::grpc::Status grpc_status;
......@@ -61,9 +61,8 @@ GrpcRequestHandler::DropTable(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context,
const ::milvus::grpc::IndexParam *request,
::milvus::grpc::Status *response) {
const ::milvus::grpc::IndexParam *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = CreateIndexTask::Create(request);
GrpcRequestScheduler::ExecTask(task_ptr, response);
return ::grpc::Status::OK;
......@@ -71,9 +70,8 @@ GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::Insert(::grpc::ServerContext *context,
const ::milvus::grpc::InsertParam *request,
::milvus::grpc::VectorIds *response) {
const ::milvus::grpc::InsertParam *request,
::milvus::grpc::VectorIds *response) {
BaseTaskPtr task_ptr = InsertTask::Create(request, response);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -86,7 +84,6 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context,
GrpcRequestHandler::Search(::grpc::ServerContext *context,
const ::milvus::grpc::SearchParam *request,
::milvus::grpc::TopKQueryResultList *response) {
std::vector<std::string> file_id_array;
BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response);
::milvus::grpc::Status grpc_status;
......@@ -100,7 +97,6 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context,
GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context,
const ::milvus::grpc::SearchInFilesParam *request,
::milvus::grpc::TopKQueryResultList *response) {
std::vector<std::string> file_id_array;
for (int i = 0; i < request->file_id_array_size(); i++) {
file_id_array.push_back(request->file_id_array(i));
......@@ -118,7 +114,6 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context,
GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::TableSchema *response) {
BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -129,9 +124,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::CountTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::TableRowCount *response) {
const ::milvus::grpc::TableName *request,
::milvus::grpc::TableRowCount *response) {
int64_t row_count = 0;
BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count);
::milvus::grpc::Status grpc_status;
......@@ -146,7 +140,6 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context,
GrpcRequestHandler::ShowTables(::grpc::ServerContext *context,
const ::milvus::grpc::Command *request,
::milvus::grpc::TableNameList *response) {
BaseTaskPtr task_ptr = ShowTablesTask::Create(response);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -157,9 +150,8 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::Cmd(::grpc::ServerContext *context,
const ::milvus::grpc::Command *request,
::milvus::grpc::StringReply *response) {
const ::milvus::grpc::Command *request,
::milvus::grpc::StringReply *response) {
std::string result;
BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result);
::milvus::grpc::Status grpc_status;
......@@ -172,8 +164,8 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context,
const ::milvus::grpc::DeleteByRangeParam *request,
::milvus::grpc::Status *response) {
const ::milvus::grpc::DeleteByRangeParam *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -184,8 +176,8 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) {
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name());
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -196,8 +188,8 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::IndexParam *response) {
const ::milvus::grpc::TableName *request,
::milvus::grpc::IndexParam *response) {
BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response);
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -208,8 +200,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context,
::grpc::Status
GrpcRequestHandler::DropIndex(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) {
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) {
BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name());
::milvus::grpc::Status grpc_status;
GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status);
......@@ -218,8 +210,7 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext *context,
return ::grpc::Status::OK;
}
}
}
}
}
\ No newline at end of file
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcRequestHandler.h
浏览文件 @ dd29a72d
......@@ -20,15 +20,15 @@
#include <cstdint>
#include <string>
#include "milvus.grpc.pb.h"
#include "status.pb.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
namespace zilliz {
namespace milvus {
namespace server {
namespace grpc {
class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
public:
public:
/**
* @brief Create table method
*
......@@ -103,8 +103,7 @@ public:
*/
::grpc::Status
CreateIndex(::grpc::ServerContext *context,
const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override;
const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override;
/**
* @brief Insert vector array to table
......@@ -123,8 +122,8 @@ public:
*/
::grpc::Status
Insert(::grpc::ServerContext *context,
const ::milvus::grpc::InsertParam *request,
::milvus::grpc::VectorIds *response) override;
const ::milvus::grpc::InsertParam *request,
::milvus::grpc::VectorIds *response) override;
/**
* @brief Query vector
......@@ -213,8 +212,8 @@ public:
*/
::grpc::Status
CountTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::TableRowCount *response) override;
const ::milvus::grpc::TableName *request,
::milvus::grpc::TableRowCount *response) override;
/**
* @brief List all tables in database
......@@ -253,8 +252,8 @@ public:
*/
::grpc::Status
Cmd(::grpc::ServerContext *context,
const ::milvus::grpc::Command *request,
::milvus::grpc::StringReply *response) override;
const ::milvus::grpc::Command *request,
::milvus::grpc::StringReply *response) override;
/**
* @brief delete table by range
......@@ -291,8 +290,8 @@ public:
*/
::grpc::Status
PreloadTable(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) override;
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) override;
/**
* @brief Describe index
......@@ -310,8 +309,8 @@ public:
*/
::grpc::Status
DescribeIndex(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::IndexParam *response) override;
const ::milvus::grpc::TableName *request,
::milvus::grpc::IndexParam *response) override;
/**
* @brief Drop index
......@@ -329,12 +328,12 @@ public:
*/
::grpc::Status
DropIndex(::grpc::ServerContext *context,
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) override;
const ::milvus::grpc::TableName *request,
::milvus::grpc::Status *response) override;
};
}
}
}
}
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
浏览文件 @ dd29a72d
......@@ -15,101 +15,107 @@
// specific language governing permissions and limitations
// under the License.
#include "GrpcRequestScheduler.h"
#include "server/grpc_impl/GrpcRequestScheduler.h"
#include "utils/Log.h"
#include "src/grpc/gen-status/status.pb.h"
#include "grpc/gen-status/status.pb.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace server {
namespace grpc {
using namespace ::milvus;
namespace {
::milvus::grpc::ErrorCode ErrorMap(ErrorCode code) {
static const std::map<ErrorCode, ::milvus::grpc::ErrorCode> code_map = {
{SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND},
{SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER},
{SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE},
{SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER},
{SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE},
{SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME},
{SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE},
{SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK},
{SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST},
{SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE},
{SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID},
{SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT},
{SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED},
{DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED},
{SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR},
{SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY},
};
if(code_map.find(code) != code_map.end()) {
return code_map.at(code);
} else {
return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR;
}
::milvus::grpc::ErrorCode
ErrorMap(ErrorCode code) {
static const std::map<ErrorCode, ::milvus::grpc::ErrorCode> code_map = {
{SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND},
{SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER},
{SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE},
{SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER},
{SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE},
{SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME},
{SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION},
{SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE},
{SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD},
{SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK},
{SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST},
{SERVER_INVALID_INDEX_METRIC_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE},
{SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID},
{SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT},
{SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED},
{DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED},
{SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR},
{SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY},
};
if (code_map.find(code) != code_map.end()) {
return code_map.at(code);
} else {
return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR;
}
}
} // namespace
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async)
: task_group_(task_group),
async_(async),
done_(false) {
: task_group_(task_group),
async_(async),
done_(false) {
}
GrpcBaseTask::~GrpcBaseTask() {
WaitToFinish();
}
Status GrpcBaseTask::Execute() {
Status
GrpcBaseTask::Execute() {
status_ = OnExecute();
Done();
return status_;
}
void GrpcBaseTask::Done() {
void
GrpcBaseTask::Done() {
done_ = true;
finish_cond_.notify_all();
}
Status GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) {
Status
GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) {
status_ = Status(error_code, error_msg);
SERVER_LOG_ERROR << error_msg;
return status_;
}
Status GrpcBaseTask::WaitToFinish() {
Status
GrpcBaseTask::WaitToFinish() {
std::unique_lock<std::mutex> lock(finish_mtx_);
finish_cond_.wait(lock, [this] { return done_; });
finish_cond_.wait(lock, [this] {
return done_;
});
return status_;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
GrpcRequestScheduler::GrpcRequestScheduler()
: stopped_(false) {
: stopped_(false) {
Start();
}
......@@ -117,7 +123,8 @@ GrpcRequestScheduler::~GrpcRequestScheduler() {
Stop();
}
void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) {
void
GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) {
if (task_ptr == nullptr) {
return;
}
......@@ -127,7 +134,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu
if (!task_ptr->IsAsync()) {
task_ptr->WaitToFinish();
const Status& status = task_ptr->status();
const Status &status = task_ptr->status();
if (!status.ok()) {
grpc_status->set_reason(status.message());
grpc_status->set_error_code(ErrorMap(status.code()));
......@@ -135,7 +142,8 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu
}
}
void GrpcRequestScheduler::Start() {
void
GrpcRequestScheduler::Start() {
if (!stopped_) {
return;
}
......@@ -143,7 +151,8 @@ void GrpcRequestScheduler::Start() {
stopped_ = false;
}
void GrpcRequestScheduler::Stop() {
void
GrpcRequestScheduler::Stop() {
if (stopped_) {
return;
}
......@@ -168,7 +177,8 @@ void GrpcRequestScheduler::Stop() {
SERVER_LOG_INFO << "Scheduler stopped";
}
Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) {
Status
GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) {
if (task_ptr == nullptr) {
return Status::OK();
}
......@@ -186,8 +196,8 @@ Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) {
return task_ptr->WaitToFinish();//sync execution
}
void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
void
GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
if (task_queue == nullptr) {
return;
}
......@@ -210,7 +220,8 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
}
}
Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) {
Status
GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) {
std::lock_guard<std::mutex> lock(queue_mtx_);
std::string group_name = task_ptr->TaskGroup();
......@@ -230,7 +241,7 @@ Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) {
return Status::OK();
}
}
}
}
}
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcRequestScheduler.h
浏览文件 @ dd29a72d
......@@ -19,12 +19,14 @@
#include "utils/Status.h"
#include "utils/BlockingQueue.h"
#include "status.grpc.pb.h"
#include "status.pb.h"
#include "grpc/gen-status/status.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
#include <map>
#include <vector>
#include <thread>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -32,30 +34,36 @@ namespace server {
namespace grpc {
class GrpcBaseTask {
protected:
GrpcBaseTask(const std::string &task_group, bool async = false);
protected:
explicit GrpcBaseTask(const std::string &task_group, bool async = false);
virtual ~GrpcBaseTask();
public:
public:
Status Execute();
void Done();
Status WaitToFinish();
std::string TaskGroup() const { return task_group_; }
std::string TaskGroup() const {
return task_group_;
}
const Status& status() const { return status_; }
const Status &status() const {
return status_;
}
bool IsAsync() const { return async_; }
bool IsAsync() const {
return async_;
}
protected:
protected:
virtual Status OnExecute() = 0;
Status SetStatus(ErrorCode error_code, const std::string &msg);
protected:
protected:
mutable std::mutex finish_mtx_;
std::condition_variable finish_cond_;
......@@ -71,7 +79,7 @@ using TaskQueuePtr = std::shared_ptr<TaskQueue>;
using ThreadPtr = std::shared_ptr<std::thread>;
class GrpcRequestScheduler {
public:
public:
static GrpcRequestScheduler &GetInstance() {
static GrpcRequestScheduler scheduler;
return scheduler;
......@@ -85,7 +93,7 @@ public:
static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status);
protected:
protected:
GrpcRequestScheduler();
virtual ~GrpcRequestScheduler();
......@@ -94,7 +102,7 @@ protected:
Status PutTaskToQueue(const BaseTaskPtr &task_ptr);
private:
private:
mutable std::mutex queue_mtx_;
std::map<std::string, TaskQueuePtr> task_groups_;
......@@ -104,7 +112,7 @@ private:
bool stopped_;
};
}
}
}
}
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcRequestTask.h
浏览文件 @ dd29a72d
......@@ -16,15 +16,18 @@
// under the License.
#pragma once
#include "GrpcRequestScheduler.h"
#include "server/grpc_impl/GrpcRequestScheduler.h"
#include "utils/Status.h"
#include "db/Types.h"
#include "milvus.grpc.pb.h"
#include "status.pb.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
#include <condition_variable>
#include <memory>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
......@@ -33,138 +36,130 @@ namespace grpc {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const ::milvus::grpc::TableSchema *schema);
protected:
explicit
CreateTableTask(const ::milvus::grpc::TableSchema *request);
protected:
explicit CreateTableTask(const ::milvus::grpc::TableSchema *request);
Status
OnExecute() override;
private:
private:
const ::milvus::grpc::TableSchema *schema_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class HasTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name, bool &has_table);
protected:
protected:
HasTableTask(const std::string &request, bool &has_table);
Status
OnExecute() override;
private:
private:
std::string table_name_;
bool &has_table_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DescribeTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema);
protected:
protected:
DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema);
Status
OnExecute() override;
private:
private:
std::string table_name_;
::milvus::grpc::TableSchema *schema_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DropTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name);
protected:
explicit
DropTableTask(const std::string &table_name);
protected:
explicit DropTableTask(const std::string &table_name);
Status
OnExecute() override;
private:
private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateIndexTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const ::milvus::grpc::IndexParam *index_Param);
protected:
explicit
CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param);
protected:
explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param);
Status
OnExecute() override;
private:
private:
const ::milvus::grpc::IndexParam *index_param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowTablesTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(::milvus::grpc::TableNameList *table_name_list);
protected:
explicit
ShowTablesTask(::milvus::grpc::TableNameList *table_name_list);
protected:
explicit ShowTablesTask(::milvus::grpc::TableNameList *table_name_list);
Status
OnExecute() override;
private:
private:
::milvus::grpc::TableNameList *table_name_list_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class InsertTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const ::milvus::grpc::InsertParam *insert_Param,
::milvus::grpc::VectorIds *record_ids_);
protected:
protected:
InsertTask(const ::milvus::grpc::InsertParam *insert_Param,
::milvus::grpc::VectorIds *record_ids_);
::milvus::grpc::VectorIds *record_ids_);
Status
OnExecute() override;
private:
private:
const ::milvus::grpc::InsertParam *insert_param_;
::milvus::grpc::VectorIds *record_ids_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class SearchTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const ::milvus::grpc::SearchParam *search_param,
const std::vector<std::string> &file_id_array,
::milvus::grpc::TopKQueryResultList *response);
protected:
protected:
SearchTask(const ::milvus::grpc::SearchParam *search_param,
const std::vector<std::string> &file_id_array,
::milvus::grpc::TopKQueryResultList *response);
......@@ -172,7 +167,7 @@ protected:
Status
OnExecute() override;
private:
private:
const ::milvus::grpc::SearchParam *search_param_;
std::vector<std::string> file_id_array_;
::milvus::grpc::TopKQueryResultList *topk_result_list;
......@@ -180,107 +175,106 @@ private:
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CountTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name, int64_t &row_count);
protected:
protected:
CountTableTask(const std::string &table_name, int64_t &row_count);
Status
OnExecute() override;
private:
private:
std::string table_name_;
int64_t &row_count_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CmdTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &cmd, std::string &result);
protected:
protected:
CmdTask(const std::string &cmd, std::string &result);
Status
OnExecute() override;
private:
private:
std::string cmd_;
std::string &result_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DeleteByRangeTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param);
protected:
DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param);
protected:
explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param);
Status
OnExecute() override;
private:
private:
const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class PreloadTableTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name);
protected:
PreloadTableTask(const std::string &table_name);
protected:
explicit PreloadTableTask(const std::string &table_name);
Status
OnExecute() override;
private:
private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DescribeIndexTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name,
::milvus::grpc::IndexParam *index_param);
::milvus::grpc::IndexParam *index_param);
protected:
protected:
DescribeIndexTask(const std::string &table_name,
::milvus::grpc::IndexParam *index_param);
::milvus::grpc::IndexParam *index_param);
Status
OnExecute() override;
private:
private:
std::string table_name_;
::milvus::grpc::IndexParam *index_param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DropIndexTask : public GrpcBaseTask {
public:
public:
static BaseTaskPtr
Create(const std::string &table_name);
protected:
DropIndexTask(const std::string &table_name);
protected:
explicit DropIndexTask(const std::string &table_name);
Status
OnExecute() override;
private:
private:
std::string table_name_;
};
}
}
}
}
\ No newline at end of file
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @ dd29a72d
......@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include "milvus.grpc.pb.h"
#include "GrpcServer.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "server/grpc_impl/GrpcServer.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "utils/Log.h"
......@@ -28,6 +28,7 @@
#include <random>
#include <string>
#include <thread>
#include <vector>
#include <grpc/grpc.h>
#include <grpcpp/channel.h>
......@@ -35,14 +36,12 @@
#include <grpcpp/create_channel.h>
#include <grpcpp/security/credentials.h>
namespace zilliz {
namespace milvus {
namespace server {
namespace grpc {
constexpr long MESSAGE_SIZE = -1;
constexpr int64_t MESSAGE_SIZE = -1;
//this class is to check port occupation during server start
class NoReusePortOption : public ::grpc::ServerBuilderOption {
......@@ -52,11 +51,9 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption {
}
void UpdatePlugins(std::vector<std::unique_ptr<::grpc::ServerBuilderPlugin>> *plugins) override {
}
};
void
GrpcServer::Start() {
thread_ptr_ = std::make_shared<std::thread>(&GrpcServer::StartService, this);
......@@ -117,7 +114,7 @@ GrpcServer::StopService() {
return Status::OK();
}
}
}
}
}
\ No newline at end of file
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
cpp/src/server/grpc_impl/GrpcServer.h
浏览文件 @ dd29a72d
......@@ -19,12 +19,12 @@
#include "utils/Status.h"
#include <memory>
#include <cstdint>
#include <string>
#include <thread>
#include <grpcpp/grpcpp.h>
namespace zilliz {
namespace milvus {
namespace server {
......@@ -52,7 +52,7 @@ class GrpcServer {
std::shared_ptr<std::thread> thread_ptr_;
};
}
}
}
}
} // namespace grpc
} // namespace server
} // namespace milvus
} // namespace zilliz
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或