提交 66f33096 编写于 作者: G groot 提交者: jinhai

add server code


Former-commit-id: 33aeca704be0e7e80f1e45bb1dc11d1aafd4f4f3
上级 dafd5a45
......@@ -52,6 +52,9 @@ include_directories(${VECWISE_ENGINE_INCLUDE})
include_directories(${VECWISE_ENGINE_SRC})
include_directories(${VECWISE_THIRD_PARTY_BUILD}/include)
link_directories(${CMAKE_CURRRENT_BINARY_DIR})
link_directories(./third_party/build/lib)
execute_process(COMMAND bash build.sh
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
......@@ -60,3 +63,5 @@ add_subdirectory(src)
if (BUILD_UNIT_TEST)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
endif(BUILD_UNIT_TEST)
add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
server_config:
address: 127.0.0.1
port: 21001
\ No newline at end of file
......@@ -5,10 +5,27 @@
#-------------------------------------------------------------------------------
aux_source_directory(./cache cache_files)
aux_source_directory(./config config_files)
aux_source_directory(./server server_files)
aux_source_directory(./utils utils_files)
set(vecwise_engine_src
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
${cache_files}
cache/DataObj.h)
)
add_library(vecwise_engine SHARED ${vecwise_engine_src})
\ No newline at end of file
add_library(vecwise_engine SHARED ${vecwise_engine_src})
add_executable(vecwise_engine_server
${config_files}
${server_files}
${utils_files}
)
set(dependency_libs
vecwise_engine
yaml-cpp
boost_system
boost_filesystem
)
target_link_libraries(vecwise_engine_server ${dependency_libs})
\ No newline at end of file
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "ConfigNode.h"
#include "utils/Error.h"
#include "utils/CommonUtil.h"
#include <sstream>
#include <string>
#include <algorithm>
namespace zilliz {
namespace vecwise {
namespace server {
void ConfigNode::Combine(const ConfigNode& target) {
const std::map<std::string, std::string>& kv = target.GetConfig();
for(auto itr = kv.begin(); itr != kv.end(); ++itr){
config_[itr->first] = itr->second;
}
const std::map<std::string, std::vector<std::string> >& sequences = target.GetSequences();
for(auto itr = sequences.begin(); itr != sequences.end(); ++itr){
sequences_[itr->first] = itr->second;
}
const std::map<std::string, ConfigNode>& children = target.GetChildren();
for(auto itr = children.begin(); itr != children.end(); ++itr){
children_[itr->first] = itr->second;
}
}
//key/value pair config
void
ConfigNode::SetValue(const std::string& key, const std::string& value) {
config_[key] = value;
}
std::string
ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const {
auto ref = config_.find(param_key);
if(ref != config_.end()) {
return ref->second;
}
//THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
return default_val;
}
bool
ConfigNode::GetBoolValue(const std::string &param_key, bool default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
std::transform(val.begin(), val.end(), val.begin(), ::tolower);
return (val == "true" || val == "on" || val == "yes" || val == "1");
} else {
return default_val;
}
}
int32_t
ConfigNode::GetInt32Value(const std::string &param_key, int32_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return (int32_t)std::strtol(val.c_str(), nullptr, 10);
} else {
return default_val;
}
}
int64_t
ConfigNode::GetInt64Value(const std::string &param_key, int64_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtol(val.c_str(), nullptr, 10);
} else {
return default_val;
}
}
float
ConfigNode::GetFloatValue(const std::string &param_key, float default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtof(val.c_str(), nullptr);
} else {
return default_val;
}
}
double
ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtold(val.c_str(), nullptr);
} else {
return default_val;
}
}
const std::map<std::string, std::string>&
ConfigNode::GetConfig() const {
return config_;
};
void ConfigNode::ClearConfig() {
config_.clear();
}
//key/object config
void
ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) {
children_[type_name] = config;
}
ConfigNode
ConfigNode::GetChild(const std::string& type_name) const {
auto ref = children_.find(type_name);
if(ref != children_.end()) {
return ref->second;
}
ConfigNode nc;
return nc;
}
ConfigNode&
ConfigNode::GetChild(const std::string &type_name) {
return children_[type_name];
}
void
ConfigNode::GetChildren(ConfigNodeArr& arr) const {
arr.clear();
for(auto ref : children_){
arr.push_back(ref.second);
}
}
const std::map<std::string, ConfigNode>&
ConfigNode::GetChildren() const {
return children_;
}
void ConfigNode::ClearChildren() {
children_.clear();
}
//key/sequence config
void
ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) {
sequences_[key].push_back(item);
}
std::vector<std::string>
ConfigNode::GetSequence(const std::string &key) const {
auto itr = sequences_.find(key);
if(itr != sequences_.end()) {
return itr->second;
} else {
std::vector<std::string> temp;
return temp;
}
}
const std::map<std::string, std::vector<std::string> >&
ConfigNode::GetSequences() const {
return sequences_;
}
void ConfigNode::ClearSequences() {
sequences_.clear();
}
void
ConfigNode::PrintAll(const std::string& prefix) const {
for(auto& elem : config_) {
CommonUtil::PrintInfo(prefix + elem.first + ": " + elem.second);
}
for(auto& elem : sequences_) {
CommonUtil::PrintInfo(prefix + elem.first + ": ");
for(auto& str : elem.second) {
CommonUtil::PrintInfo(prefix + " - " + str);
}
}
for(auto& elem : children_) {
CommonUtil::PrintInfo(prefix + elem.first + ": ");
elem.second.PrintAll(prefix + " ");
}
}
std::string
ConfigNode::DumpString(const std::string &prefix) const {
std::stringstream str_buffer;
const std::string endl = "\n";
for(auto& elem : config_) {
str_buffer << prefix << elem.first << ": " << elem.second << endl;
}
for(auto& elem : sequences_) {
str_buffer << prefix << elem.first << ": " << endl;
for(auto& str : elem.second) {
str_buffer << prefix + " - " << str << endl;
}
}
for(auto& elem : children_) {
str_buffer << prefix << elem.first << ": " << endl;
str_buffer << elem.second.DumpString(prefix + " ") << endl;
}
return str_buffer.str();
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <vector>
#include <string>
#include <map>
namespace zilliz {
namespace vecwise {
namespace server {
class ConfigNode;
typedef std::vector<ConfigNode> ConfigNodeArr;
class ConfigNode {
public:
void Combine(const ConfigNode& target);
//key/value pair config
void SetValue(const std::string &key, const std::string &value);
std::string GetValue(const std::string &param_key, const std::string &default_val = "") const;
bool GetBoolValue(const std::string &param_key, bool default_val = false) const;
int32_t GetInt32Value(const std::string &param_key, int32_t default_val = 0) const;
int64_t GetInt64Value(const std::string &param_key, int64_t default_val = 0) const;
float GetFloatValue(const std::string &param_key, float default_val = 0.0) const;
double GetDoubleValue(const std::string &param_key, double default_val = 0.0) const;
const std::map<std::string, std::string>& GetConfig() const;
void ClearConfig();
//key/object config
void AddChild(const std::string &type_name, const ConfigNode &config);
ConfigNode GetChild(const std::string &type_name) const;
ConfigNode& GetChild(const std::string &type_name);
void GetChildren(ConfigNodeArr &arr) const;
const std::map<std::string, ConfigNode>& GetChildren() const;
void ClearChildren();
//key/sequence config
void AddSequenceItem(const std::string &key, const std::string &item);
std::vector<std::string> GetSequence(const std::string &key) const;
const std::map<std::string, std::vector<std::string> >& GetSequences() const;
void ClearSequences();
void PrintAll(const std::string &prefix = "") const;
std::string DumpString(const std::string &prefix = "") const;
private:
std::map<std::string, std::string> config_;
std::map<std::string, ConfigNode> children_;
std::map<std::string, std::vector<std::string> > sequences_;
};
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "IConfigMgr.h"
#include "YamlConfigMgr.h"
namespace zilliz {
namespace vecwise {
namespace server {
IConfigMgr * IConfigMgr::GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include "ConfigNode.h"
namespace zilliz {
namespace vecwise {
namespace server {
// this class can parse nested config file and return config item
// config file example(yaml style)
// AAA: 1
// BBB:
// CCC: hello
// DDD: 23.5
//
// usage
// const IConfigMgr* mgr = IConfigMgr::GetInstance();
// const ConfigNode& node = mgr->GetRootNode();
// std::string val = node.GetValue("AAA"); // return '1'
// const ConfigNode& child = node.GetChild("BBB");
// val = child.GetValue("CCC"); //return 'hello'
class IConfigMgr {
public:
static IConfigMgr* GetInstance();
virtual ServerError LoadConfigFile(const std::string &filename) = 0;
virtual void Print() const = 0;//will be deleted
virtual std::string DumpString() const = 0;
virtual const ConfigNode& GetRootNode() const = 0;
virtual ConfigNode& GetRootNode() = 0;
};
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "YamlConfigMgr.h"
#include "utils/CommonUtil.h"
#include <sys/stat.h>
namespace zilliz {
namespace vecwise {
namespace server {
ServerError YamlConfigMgr::LoadConfigFile(const std::string &filename) {
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
CommonUtil::PrintError("File not found: " + filename);
return SERVER_UNEXPECTED_ERROR;
}
try {
node_ = YAML::LoadFile(filename);
LoadConfigNode(node_, config_);
}
catch (YAML::Exception& e) {
CommonUtil::PrintError("Failed to load config file: " + std::string(e.what ()));
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
void YamlConfigMgr::Print() const {
CommonUtil::PrintInfo("System config content:");
config_.PrintAll();
}
std::string YamlConfigMgr::DumpString() const {
return config_.DumpString("");
}
const ConfigNode& YamlConfigMgr::GetRootNode() const {
return config_;
}
ConfigNode& YamlConfigMgr::GetRootNode() {
return config_;
}
bool
YamlConfigMgr::SetConfigValue(const YAML::Node& node,
const std::string& key,
ConfigNode& config) {
if(node[key].IsDefined ()) {
config.SetValue(key, node[key].as<std::string>());
return true;
}
return false;
}
bool
YamlConfigMgr::SetChildConfig(const YAML::Node& node,
const std::string& child_name,
ConfigNode& config) {
if(node[child_name].IsDefined ()) {
ConfigNode sub_config;
LoadConfigNode(node[child_name], sub_config);
config.AddChild(child_name, sub_config);
return true;
}
return false;
}
bool
YamlConfigMgr::SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
if(node[child_name].IsDefined ()) {
size_t cnt = node[child_name].size();
for(size_t i = 0; i < cnt; i++){
config.AddSequenceItem(child_name, node[child_name][i].as<std::string>());
}
return true;
}
return false;
}
void
YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
std::string key;
for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) {
if(!it->first.IsNull()){
key = it->first.as<std::string>();
}
if(node[key].IsScalar()) {
SetConfigValue(node, key, config);
} else if(node[key].IsMap()){
SetChildConfig(node, key, config);
} else if(node[key].IsSequence()){
SetSequence(node, key, config);
}
}
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "IConfigMgr.h"
#include "ConfigNode.h"
#include "utils/Error.h"
#include <yaml-cpp/yaml.h>
namespace zilliz {
namespace vecwise {
namespace server {
class YamlConfigMgr : public IConfigMgr {
public:
virtual ServerError LoadConfigFile(const std::string &filename);
virtual void Print() const;
virtual std::string DumpString() const;
virtual const ConfigNode& GetRootNode() const;
virtual ConfigNode& GetRootNode();
private:
bool SetConfigValue(const YAML::Node& node,
const std::string& key,
ConfigNode& config);
bool SetChildConfig(const YAML::Node& node,
const std::string &name,
ConfigNode &config);
bool
SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config);
void LoadConfigNode(const YAML::Node& node, ConfigNode& config);
private:
YAML::Node node_;
ConfigNode config_;
};
}
}
}
......@@ -3,7 +3,98 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "server/Server.h"
void test() {
return ;
}
\ No newline at end of file
#include <getopt.h>
#include <libgen.h>
#include <cstring>
#include <string>
#include <signal.h>
#include "utils/SignalUtil.h"
#include "utils/CommonUtil.h"
void print_help(const std::string &app_name);
using namespace zilliz::vecwise;
int
main(int argc, char *argv[]) {
server::CommonUtil::PrintInfo("Vecwise engine server start");
// zilliz::lib::gpu::InitMemoryAllocator();
signal(SIGSEGV, server::SignalUtil::HandleSignal);
signal(SIGUSR1, server::SignalUtil::HandleSignal);
signal(SIGUSR2, server::SignalUtil::HandleSignal);
std::string app_name = basename(argv[0]);
static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'},
{"help", no_argument, 0, 'h'},
{"daemon", no_argument, 0, 'd'},
{"pid_file", required_argument, 0, 'p'},
{NULL, 0, 0, 0}};
int option_index = 0;
int64_t start_daemonized = 0;
// int pid_fd;
std::string config_filename;
std::string pid_filename;
app_name = argv[0];
// if(argc < 5) {
// print_help(app_name);
// return EXIT_FAILURE;
// }
int value;
while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) {
switch (value) {
case 'c': {
char *config_filename_ptr = strdup(optarg);
config_filename = config_filename_ptr;
free(config_filename_ptr);
printf("Loading configuration from: %s\n", config_filename.c_str());
break;
}
case 'p': {
char *pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
free(pid_filename_ptr);
printf("%s\n", pid_filename.c_str());
break;
}
case 'd':
start_daemonized = 1;
break;
case 'h':
print_help(app_name);
return EXIT_SUCCESS;
case '?':
print_help(app_name);
return EXIT_FAILURE;
default:
print_help(app_name);
break;
}
}
server::Server* server_ptr = server::Server::Instance();
server_ptr->Init(start_daemonized, pid_filename, config_filename);
return server_ptr->Start();
}
void
print_help(const std::string &app_name) {
printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
printf(" Options:\n");
printf(" -h --help Print this help\n");
printf(" -c --conf_file filename Read configuration from the file\n");
printf(" -d --daemon Daemonize this application\n");
printf(" -p --pid_file filename PID file used by daemonized app\n");
printf("\n");
}
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "Server.h"
#include "utils/CommonUtil.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <csignal>
#include <numaif.h>
#include <unistd.h>
#include <string.h>
namespace zilliz {
namespace vecwise {
namespace server {
Server*
Server::Instance() {
static Server server;
return &server;
}
Server::Server()
: opt_config_ptr_(nullptr){
}
Server::~Server() {
}
void
Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename) {
daemonized_ = daemonized;
pid_filename_ = pid_filename;
config_filename_ = config_filename;
}
void
Server::Daemonize() {
if (daemonized_ == 0) {
return;
}
CommonUtil::PrintInfo("Megawise server run in daemonize mode");
// std::string log_path(GetLogDirFullPath());
// log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)";
// CommonUtil::PrintInfo("Log will be exported to: " + log_path);
pid_t pid = 0;
// Fork off the parent process
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Success: terminate parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// On success: The child process becomes session leader
if (setsid() < 0) {
exit(EXIT_FAILURE);
}
// Ignore signal sent from child to parent process
signal(SIGCHLD, SIG_IGN);
// Fork off for the second time
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Terminate the parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// Set new file permissions
umask(0);
// Change the working directory to root
int ret = chdir("/");
if(ret != 0){
return;
}
// Close all open fd
for (long fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) {
close(fd);
}
CommonUtil::PrintInfo("Redirect stdin/stdout/stderr to /dev/null");
// Redirect stdin/stdout/stderr to /dev/null
stdin = fopen("/dev/null", "r");
stdout = fopen("/dev/null", "w+");
stderr = fopen("/dev/null", "w+");
// Try to write PID of daemon to lockfile
if (!pid_filename_.empty()) {
pid_fd = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640);
if (pid_fd < 0) {
CommonUtil::PrintInfo("Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno));
exit(EXIT_FAILURE);
}
if (lockf(pid_fd, F_TLOCK, 0) < 0) {
CommonUtil::PrintInfo("Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno));
exit(EXIT_FAILURE);
}
std::string pid_file_context = std::to_string(getpid());
ssize_t res = write(pid_fd, pid_file_context.c_str(), pid_file_context.size());
if(res != 0){
return;
}
}
}
int
Server::Start() {
if (daemonized_) {
Daemonize();
}
do {
try {
// Read config file
if(LoadConfig() != SERVER_SUCCESS) {
return 1;
}
//log path is defined by LoadConfig, so InitLog must be called after LoadConfig
ServerConfig *config = ServerConfig::GetInstance();
ConfigNode server_config = config->GetServerConfig();
//print config into console and log
opt_config_ptr_->PrintAll();
// Handle Signal
signal(SIGINT, SignalUtil::HandleSignal);
signal(SIGHUP, SignalUtil::HandleSignal);
signal(SIGTERM, SignalUtil::HandleSignal);
StartService();
CommonUtil::PrintInfo("Megawise server is running...");
} catch(std::exception& ex){
std::string info = "Megawise server encounter exception: " + std::string(ex.what());
CommonUtil::PrintError(info);
CommonUtil::PrintInfo("Is another server instance running?");
break;
}
} while(false);
Stop();
return 0;
}
void
Server::Stop() {
CommonUtil::PrintInfo("Megawise server will be closed");
// Unlock and close lockfile
if (pid_fd != -1) {
int ret = lockf(pid_fd, F_ULOCK, 0);
if(ret != 0){
}
ret = close(pid_fd);
if(ret != 0){
}
}
// Try to delete lockfile
if (!pid_filename_.empty()) {
int ret = unlink(pid_filename_.c_str());
if(ret != 0){
}
}
running_ = 0;
StopService();
CommonUtil::PrintInfo("Megawise server closed");
}
ServerError
Server::LoadConfig() {
opt_config_ptr_ = ServerConfig::GetInstance();
opt_config_ptr_->LoadConfigFile(config_filename_);
return SERVER_SUCCESS;
}
void
Server::StartService() {
}
void
Server::StopService() {
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "ServerConfig.h"
#include "utils/Error.h"
#include <cstdint>
#include <string>
namespace zilliz {
namespace vecwise {
namespace server {
class Server {
public:
static Server* Instance();
void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename);
int Start();
void Stop();
private:
Server();
~Server();
void Daemonize();
static void HandleSignal(int signal);
ServerError LoadConfig();
void StartService();
void StopService();
private:
int64_t daemonized_ = 0;
int64_t running_ = 1;
int pid_fd = -1;
std::string pid_filename_;
std::string config_filename_;
ServerConfig* opt_config_ptr_ = nullptr;
}; // Server
} // server
} // sql
} // zilliz
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "ServerConfig.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <iostream>
#include "config/IConfigMgr.h"
namespace zilliz {
namespace vecwise {
namespace server {
ServerConfig*
ServerConfig::GetInstance() {
static ServerConfig config;
return &config;
}
ServerError
ServerConfig::LoadConfigFile(const std::string& config_filename) {
std::string filename = config_filename;
if(filename.empty()){
std::cout << "ERROR: a config file is required" << std::endl;
exit(1);//directly exit program if config file not specified
}
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
std::cout << "ERROR: " << filename << " not found!" << std::endl;
exit(1);//directly exit program if config file not found
}
try {
IConfigMgr* mgr = const_cast<IConfigMgr*>(IConfigMgr::GetInstance());
ServerError err = mgr->LoadConfigFile(filename);
if(err != 0) {
std::cout << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal
}
}
catch (YAML::Exception& e) {
std::cout << "Server failed to load config file: " << std::endl;
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
void
ServerConfig::PrintAll() const {
if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) {
std::string str = mgr->DumpString();
// SERVER_LOG_INFO << "\n" << str;
std::cout << "\n" << str << std::endl;
}
}
ConfigNode
ServerConfig::GetServerConfig() const {
const IConfigMgr* mgr = IConfigMgr::GetInstance();
const ConfigNode& root_node = mgr->GetRootNode();
return root_node.GetChild(CONFIG_SERVER);
}
ConfigNode&
ServerConfig::GetServerConfig() {
IConfigMgr* mgr = IConfigMgr::GetInstance();
ConfigNode& root_node = mgr->GetRootNode();
return root_node.GetChild(CONFIG_SERVER);
}
std::string
ServerConfig::GetServerAddress() const {
ConfigNode server_config = GetServerConfig();
return server_config.GetValue(CONFIG_ADDRESS);
}
std::string
ServerConfig::GetServerPort() const {
ConfigNode server_config = GetServerConfig();
return server_config.GetValue(CONFIG_PORT);
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include "config/ConfigNode.h"
#include <yaml-cpp/yaml.h>
namespace zilliz {
namespace vecwise {
namespace server {
static const std::string CONFIG_SERVER = "server_config";
static const std::string CONFIG_ADDRESS = "address";
static const std::string CONFIG_PORT = "port";
class ServerConfig {
public:
static ServerConfig *GetInstance();
ServerError LoadConfigFile(const std::string& config_filename);
void PrintAll() const;
ConfigNode GetServerConfig() const;
ConfigNode& GetServerConfig();
std::string GetServerAddress() const;
std::string GetServerPort() const;
};
}
}
}
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "CommonUtil.h"
#include <unistd.h>
#include <sys/sysinfo.h>
#include <pwd.h>
#include <thread>
#include <sys/stat.h>
#include <dirent.h>
#include <string.h>
#include <iostream>
#include "boost/filesystem.hpp"
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
namespace zilliz {
namespace vecwise {
namespace server {
namespace fs = boost::filesystem;
void CommonUtil::PrintInfo(const std::string& info){
// SERVER_LOG_INFO << info;
std::cout << info << std::endl;
}
void CommonUtil::PrintError(const std::string& info){
// SERVER_LOG_ERROR << info;
std::cout << info << std::endl;
}
bool CommonUtil::GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem) {
struct sysinfo info;
int ret = sysinfo(&info);
totalMem = info.totalram;
freeMem = info.freeram;
return ret == 0;//succeed 0, failed -1
}
bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) {
//threadCnt = std::thread::hardware_concurrency();
threadCnt = sysconf(_SC_NPROCESSORS_CONF);
threadCnt *= THREAD_MULTIPLY_CPU;
if (threadCnt == 0)
threadCnt = 8;
return true;
}
bool CommonUtil::IsDirectoryExit(const std::string &path)
{
DIR *dp = nullptr;
if ((dp = opendir(path.c_str())) == nullptr) {
return false;
}
closedir(dp);
return true;
}
ServerError CommonUtil::CreateDirectory(const std::string &path) {
struct stat directoryStat;
int statOK = stat(path.c_str(), &directoryStat);
if (statOK == 0) {
return SERVER_SUCCESS;//already exist
}
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
ServerError err = CreateDirectory(parent_path.string());
if(err != SERVER_SUCCESS){
return err;
}
statOK = stat(path.c_str(), &directoryStat);
if (statOK == 0) {
return SERVER_SUCCESS;//already exist
}
int makeOK = mkdir(path.c_str(), S_IRWXU|S_IRGRP|S_IROTH);
if (makeOK != 0) {
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
void RemoveDirectory(const std::string &path) {
DIR *pDir = NULL;
struct dirent *dmsg;
char szFileName[256];
char szFolderName[256];
strcpy(szFolderName, path.c_str());
strcat(szFolderName, "/%s");
if ((pDir = opendir(path.c_str())) != NULL) {
while ((dmsg = readdir(pDir)) != NULL) {
if (strcmp(dmsg->d_name, ".") != 0
&& strcmp(dmsg->d_name, "..") != 0) {
sprintf(szFileName, szFolderName, dmsg->d_name);
std::string tmp = szFileName;
if (tmp.find(".") == std::string::npos) {
RemoveDirectory(szFileName);
}
remove(szFileName);
}
}
}
if (pDir != NULL) {
closedir(pDir);
}
remove(path.c_str());
}
ServerError DeleteDirectory(const std::string &path) {
struct stat directoryStat;
int statOK = stat(path.c_str(), &directoryStat);
if (statOK != 0)
return SERVER_SUCCESS;
RemoveDirectory(path);
return SERVER_SUCCESS;
}
bool IsFileExist(const std::string &path) {
return (access(path.c_str(), F_OK) == 0);
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <string>
#include "Error.h"
namespace zilliz {
namespace vecwise {
namespace server {
class CommonUtil {
public:
static void PrintInfo(const std::string& info);
static void PrintError(const std::string& info);
static bool GetSystemMemInfo(unsigned long &totalMem, unsigned long &freeMem);
static bool GetSystemAvailableThreads(unsigned int &threadCnt);
static bool IsFileExist(const std::string &path);
static bool IsDirectoryExit(const std::string &path);
static ServerError CreateDirectory(const std::string &path);
static ServerError DeleteDirectory(const std::string &path);
};
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <cstdint>
namespace zilliz {
namespace vecwise {
namespace server {
using ServerError = int32_t;
constexpr ServerError SERVER_SUCCESS = 0;
constexpr ServerError SERVER_ERROR_CODE_BASE = 0x30000;
constexpr ServerError
ToGlobalServerErrorCode(const ServerError error_code) {
return SERVER_ERROR_CODE_BASE + SERVER_ERROR_CODE_BASE;
}
constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001);
constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(0x002);
constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003);
constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004);
constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005);
constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006);
} // namespace server
} // namespace vecwise
} // namespace zilliz
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "SignalUtil.h"
#include "CommonUtil.h"
#include "server/Server.h"
#include <signal.h>
#include <execinfo.h>
namespace zilliz {
namespace vecwise {
namespace server {
void SignalUtil::HandleSignal(int signum){
CommonUtil::PrintInfo("Server received signal:" + std::to_string(signum));
switch(signum){
case SIGUSR2:{
server::Server* server_ptr = server::Server::Instance();
server_ptr->Stop();
exit(0);
}
default:{
SignalUtil::PrintStacktrace();
std::string info = "Server encounter critical signal:";
info += std::to_string(signum);
// SendSignalMessage(signum, info);
CommonUtil::PrintInfo(info);
server::Server* server_ptr = server::Server::Instance();
server_ptr->Stop();
exit(1);
}
}
}
void SignalUtil::PrintStacktrace() {
CommonUtil::PrintInfo("Call stack:");
const int size = 32;
void* array[size];
int stack_num = backtrace(array, size);
char ** stacktrace = backtrace_symbols(array, stack_num);
for (int i = 0; i < stack_num; ++i) {
std::string info = stacktrace[i];
CommonUtil::PrintInfo(info);
}
free(stacktrace);
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
namespace zilliz {
namespace vecwise {
namespace server {
class SignalUtil {
public:
static void HandleSignal(int signum);
static void PrintStacktrace();
};
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <vector>
#include <queue>
#include <memory>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <future>
#include <functional>
#include <stdexcept>
#define MAX_THREADS_NUM 32
namespace zilliz {
namespace sql {
namespace storage {
class ThreadPool {
public:
ThreadPool(size_t threads, size_t queue_size = 1000);
template<class F, class... Args>
auto enqueue(F &&f, Args &&... args)
-> std::future<typename std::result_of<F(Args...)>::type>;
~ThreadPool();
private:
// need to keep track of threads so we can join them
std::vector<std::thread> workers;
// the task queue
std::queue<std::function<void()> > tasks;
size_t max_queue_size;
// synchronization
std::mutex queue_mutex;
std::condition_variable condition;
bool stop;
};
// the constructor just launches some amount of workers
inline ThreadPool::ThreadPool(size_t threads, size_t queue_size)
: max_queue_size(queue_size), stop(false) {
for (size_t i = 0; i < threads; ++i)
workers.emplace_back(
[this] {
for (;;) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock(this->queue_mutex);
this->condition.wait(lock,
[this] { return this->stop || !this->tasks.empty(); });
if (this->stop && this->tasks.empty())
return;
task = std::move(this->tasks.front());
this->tasks.pop();
}
this->condition.notify_all();
task();
}
}
);
}
// add new work item to the pool
template<class F, class... Args>
auto ThreadPool::enqueue(F &&f, Args &&... args)
-> std::future<typename std::result_of<F(Args...)>::type> {
using return_type = typename std::result_of<F(Args...)>::type;
auto task = std::make_shared<std::packaged_task<return_type()> >(
std::bind(std::forward<F>(f), std::forward<Args>(args)...)
);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex);
this->condition.wait(lock,
[this] { return this->tasks.size() < max_queue_size; });
// don't allow enqueueing after stopping the pool
if (stop)
throw std::runtime_error("enqueue on stopped ThreadPool");
tasks.emplace([task]() { (*task)(); });
}
condition.notify_all();
return res;
}
// the destructor joins all threads
inline ThreadPool::~ThreadPool() {
{
std::unique_lock<std::mutex> lock(queue_mutex);
stop = true;
}
condition.notify_all();
for (std::thread &worker: workers)
worker.join();
}
}
}
}
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "TimeRecorder.h"
#include "utils/CommonUtil.h"
namespace zilliz {
namespace vecwise {
namespace server {
TimeRecorder::TimeRecorder(const std::string &header,
TimeRecorder::TimeDisplayUnit unit,
int64_t log_level) :
header_(header),
time_unit_(unit),
log_level_(log_level) {
start_ = last_ = stdclock::now();
span_ = 0.0;
}
std::string
TimeRecorder::GetTimeSpanStr(TimeRecorder::TimeDisplayUnit &unit, double span) const {
std::string spanStr;
std::string unitStr;
switch (unit) {
case TimeRecorder::eTimeAutoUnit: {
if (span >= 1000000) {
int64_t t = (int64_t) span;
int64_t hour, minute;
double second;
hour = t / 1000000 / 3600;
t -= hour * 3600 * 1000000;
minute = t / 1000000 / 60;
t -= minute * 60 * 1000000;
second = t * 0.000001;
spanStr += (hour < 10 ? "0" : "") + std::to_string(hour) + ":";
spanStr += (minute < 10 ? "0" : "") + std::to_string(minute) + ":";
spanStr += (second < 10 ? "0" : "") + std::to_string(second);
unitStr = "";
} else if (span >= 1000) {
spanStr = std::to_string(span * 0.001);
unitStr = " ms";
} else {
spanStr = std::to_string(span);
unitStr = " us";
}
}
break;
case TimeRecorder::eTimeHourUnit:
spanStr = std::to_string((span * 0.000001) / 3600);
unitStr = " hour";
break;
case TimeRecorder::eTimeMinuteUnit:
spanStr = std::to_string((span * 0.000001) / 60);
unitStr = " min";
break;
case TimeRecorder::eTimeSecondUnit:
spanStr = std::to_string(span * 0.000001);
unitStr = " sec";
break;
case TimeRecorder::eTimeMilliSecUnit:
spanStr = std::to_string(span * 0.001);
unitStr = " ms";
break;
case TimeRecorder::eTimeMicroSecUnit:
default:
spanStr = std::to_string(span);
unitStr = " us";
break;
}
return spanStr + unitStr;
}
void
TimeRecorder::PrintTimeRecord(const std::string &msg, double span) {
std::string strLog;
if (!header_.empty()) strLog += header_ + ": ";
strLog += msg;
strLog += " (";
strLog += GetTimeSpanStr(time_unit_, span);
strLog += ")";
switch (log_level_) {
case 0: {
CommonUtil::PrintInfo(strLog);
break;
}
case 1: {
CommonUtil::PrintInfo(strLog);
break;
}
case 2: {
CommonUtil::PrintInfo(strLog);
break;
}
case 3: {
CommonUtil::PrintInfo(strLog);
break;
}
case 4: {
CommonUtil::PrintInfo(strLog);
break;
}
case 5: {
CommonUtil::PrintInfo(strLog);
break;
}
default: {
CommonUtil::PrintInfo(strLog);
break;
}
}
}
void
TimeRecorder::Record(const std::string &msg) {
stdclock::time_point curr = stdclock::now();
span_ = (std::chrono::duration<double, std::micro>(curr - last_)).count();
last_ = curr;
PrintTimeRecord(msg, span_);
}
void
TimeRecorder::Elapse(const std::string &msg) {
stdclock::time_point curr = stdclock::now();
span_ = (std::chrono::duration<double, std::micro>(curr - start_)).count();
PrintTimeRecord(msg, span_);
}
double
TimeRecorder::Span() {
return span_;
}
}
}
}
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <string>
#include <chrono>
namespace zilliz {
namespace vecwise {
namespace server {
class TimeRecorder {
using stdclock = std::chrono::high_resolution_clock;
public:
enum TimeDisplayUnit {
eTimeAutoUnit = 0,
eTimeHourUnit,
eTimeMinuteUnit,
eTimeSecondUnit,
eTimeMilliSecUnit,
eTimeMicroSecUnit,
};
TimeRecorder(const std::string &header,
TimeRecorder::TimeDisplayUnit unit = TimeRecorder::eTimeAutoUnit,
int64_t log_level = 1); //trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5
void Record(const std::string &msg);
void Elapse(const std::string &msg);
double Span();
private:
std::string GetTimeSpanStr(TimeRecorder::TimeDisplayUnit &unit, double span) const;
void PrintTimeRecord(const std::string &msg, double span);
private:
std::string header_;
TimeRecorder::TimeDisplayUnit time_unit_;
stdclock::time_point start_;
stdclock::time_point last_;
double span_;
int64_t log_level_;
};
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册