diff --git a/core/CHANGELOG.md b/core/CHANGELOG.md index bbd58fb28d237d64b49074ff8f8ad9a7426e7981..1bdb37d6badf0997dce78085c270ee56995a9059 100644 --- a/core/CHANGELOG.md +++ b/core/CHANGELOG.md @@ -11,16 +11,17 @@ Please mark all change in change log and use the ticket from JIRA. - MS-587 - Count get wrong result after adding vectors and index built immediately - MS-599 - Search wrong result when table created with metric_type: IP - MS-601 - Docker logs error caused by get CPUTemperature error -- MS-622 - Delete vectors should be failed if date range is invalid - MS-620 - Get table row counts display wrong error code +- MS-622 - Delete vectors should be failed if date range is invalid +- MS-624 - Search vectors failed if time ranges long enough - MS-637 - Out of memory when load too many tasks +- MS-639 - SQ8H index created failed and server hang - MS-640 - Cache object size calculate incorrect - MS-641 - Segment fault(signal 11) in PickToLoad -- MS-639 - SQ8H index created failed and server hang -- MS-647 - [monitor] grafana display average cpu-temp - MS-644 - Search crashed with index-type: flat -- MS-624 - Search vectors failed if time ranges long enough +- MS-647 - [monitor] grafana display average cpu-temp - MS-652 - IVFSQH quantization double free +- MS-653 - When config check fail, Milvus close without message ## Improvement - MS-552 - Add and change the easylogging library @@ -38,8 +39,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-608 - Update TODO names - MS-609 - Update task construct function - MS-611 - Add resources validity check in ResourceMgr -- MS-619 - Add optimizer class in scheduler - MS-614 - Preload table at startup +- MS-619 - Add optimizer class in scheduler - MS-626 - Refactor DataObj to support cache any type data - MS-648 - Improve unittest @@ -59,8 +60,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-600 - Reconstruct unittest code - MS-602 - Remove zilliz namespace - MS-610 - Change error code base value from hex to decimal -- MS-635 - Add compile option to support customized faiss - MS-624 - Re-organize project directory for open-source +- MS-635 - Add compile option to support customized faiss # Milvus 0.4.0 (2019-09-12) diff --git a/core/src/config/ConfigMgr.cpp b/core/src/config/ConfigMgr.cpp deleted file mode 100644 index 5889f52fac2987ab7a8c7830c176acdf2d0417c6..0000000000000000000000000000000000000000 --- a/core/src/config/ConfigMgr.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "config/ConfigMgr.h" -#include "YamlConfigMgr.h" - -namespace milvus { -namespace server { - -ConfigMgr* -ConfigMgr::GetInstance() { - static YamlConfigMgr mgr; - return &mgr; -} - -} // namespace server -} // namespace milvus diff --git a/core/src/config/ConfigMgr.h b/core/src/config/ConfigMgr.h index 40753beaa9766e536fb2c34cecfd7166022add49..9d79e07a7028418ad89d56b31de2d85ae7628c8f 100644 --- a/core/src/config/ConfigMgr.h +++ b/core/src/config/ConfigMgr.h @@ -17,42 +17,28 @@ #pragma once -#include "ConfigNode.h" -#include "utils/Error.h" - #include +#include "ConfigNode.h" +#include "utils/Status.h" + namespace milvus { namespace server { -// this class can parse nested config file and return config item -// config file example(yaml style) -// AAA: 1 -// BBB: -// CCC: hello -// DDD: 23.5 -// -// usage -// const ConfigMgr* mgr = ConfigMgr::GetInstance(); -// const ConfigNode& node = mgr->GetRootNode(); -// std::string val = node.GetValue("AAA"); // return '1' -// const ConfigNode& child = node.GetChild("BBB"); -// val = child.GetValue("CCC"); //return 'hello' - class ConfigMgr { public: - static ConfigMgr* - GetInstance(); - - virtual ErrorCode + virtual Status LoadConfigFile(const std::string& filename) = 0; + virtual void Print() const = 0; // will be deleted + virtual std::string DumpString() const = 0; virtual const ConfigNode& GetRootNode() const = 0; + virtual ConfigNode& GetRootNode() = 0; }; diff --git a/core/src/config/YamlConfigMgr.cpp b/core/src/config/YamlConfigMgr.cpp index 71535495442851e1e3ce276cb916428064fb8458..c84d2cc729aaf5c6f549f739dce179f9ac6b1ea6 100644 --- a/core/src/config/YamlConfigMgr.cpp +++ b/core/src/config/YamlConfigMgr.cpp @@ -18,29 +18,20 @@ #include "config/YamlConfigMgr.h" #include "utils/Log.h" -#include - namespace milvus { namespace server { -ErrorCode +Status YamlConfigMgr::LoadConfigFile(const std::string& filename) { - struct stat directoryStat; - int statOK = stat(filename.c_str(), &directoryStat); - if (statOK != 0) { - SERVER_LOG_ERROR << "File not found: " << filename; - return SERVER_UNEXPECTED_ERROR; - } - try { node_ = YAML::LoadFile(filename); LoadConfigNode(node_, config_); } catch (YAML::Exception& e) { - SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what()); - return SERVER_UNEXPECTED_ERROR; + std::string str = "Exception: load config file fail: " + std::string(e.what()); + return Status(SERVER_UNEXPECTED_ERROR, str); } - return SERVER_SUCCESS; + return Status::OK(); } void diff --git a/core/src/config/YamlConfigMgr.h b/core/src/config/YamlConfigMgr.h index 1c68bc883fd814ccd491ee3b3ed0f89654c1185f..669e38017a578105c95d3f394fd2549d9c887eb2 100644 --- a/core/src/config/YamlConfigMgr.h +++ b/core/src/config/YamlConfigMgr.h @@ -17,27 +17,35 @@ #pragma once -#include "ConfigMgr.h" -#include "ConfigNode.h" -#include "utils/Error.h" - #include #include +#include "ConfigMgr.h" +#include "utils/Status.h" + namespace milvus { namespace server { class YamlConfigMgr : public ConfigMgr { public: - virtual ErrorCode + static ConfigMgr* + GetInstance() { + static YamlConfigMgr mgr; + return &mgr; + } + + virtual Status LoadConfigFile(const std::string& filename); + virtual void Print() const; + virtual std::string DumpString() const; virtual const ConfigNode& GetRootNode() const; + virtual ConfigNode& GetRootNode(); diff --git a/core/src/main.cpp b/core/src/main.cpp index b50eedeabaec2edd6dedeffb805bb07d1628c95b..769483acd3d6dd37777293e3146d463f7d3bc331 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -32,12 +32,32 @@ INITIALIZE_EASYLOGGINGPP void -print_help(const std::string& app_name); +print_help(const std::string& app_name) { + std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; + std::cout << " Options:" << std::endl; + std::cout << " -h --help Print this help" << std::endl; + std::cout << " -c --conf_file filename Read configuration from the file" << std::endl; + std::cout << " -d --daemon Daemonize this application" << std::endl; + std::cout << " -p --pid_file filename PID file used by daemonized app" << std::endl; + std::cout << std::endl; +} + +void +print_banner() { + std::cout << std::endl; + std::cout << " __ _________ _ ____ ______ " << std::endl; + std::cout << " / |/ / _/ /| | / / / / / __/ " << std::endl; + std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl; + std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; + std::cout << std::endl; + std::cout << "Welcome to use Milvus by Zilliz!" << std::endl; + std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl; + std::cout << std::endl; +} int main(int argc, char* argv[]) { - std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; - std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; + print_banner(); static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'}, {"log_conf_file", required_argument, nullptr, 'l'}, @@ -53,10 +73,12 @@ main(int argc, char* argv[]) { std::string pid_filename; std::string app_name = argv[0]; + milvus::server::Server& server = milvus::server::Server::GetInstance(); + milvus::Status s; + if (argc < 2) { print_help(app_name); - std::cout << "Milvus server exit..." << std::endl; - return EXIT_FAILURE; + goto FAIL; } int value; @@ -106,23 +128,21 @@ main(int argc, char* argv[]) { signal(SIGUSR2, milvus::server::SignalUtil::HandleSignal); signal(SIGTERM, milvus::server::SignalUtil::HandleSignal); - milvus::server::Server& server = milvus::server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - server.Start(); + + s = server.Start(); + if (s.ok()) { + std::cout << "Milvus server start successfully." << std::endl; + } else { + goto FAIL; + } /* wait signal */ pause(); - return 0; -} + return EXIT_SUCCESS; -void -print_help(const std::string& app_name) { - std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; - std::cout << " Options:" << std::endl; - std::cout << " -h --help Print this help" << std::endl; - std::cout << " -c --conf_file filename Read configuration from the file" << std::endl; - std::cout << " -d --daemon Daemonize this application" << std::endl; - std::cout << " -p --pid_file filename PID file used by daemonized app" << std::endl; - std::cout << std::endl; +FAIL: + std::cout << "Milvus server exit..." << std::endl; + return EXIT_FAILURE; } diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index b7a1e211d2b1b23dee3f903e0e7e130de1e98efe..c06580b76cbebc64e5f2d1377e15dd56ae409342 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -160,8 +160,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + - " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); // for (auto &context : search_contexts_) { diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 78a9aaad3689a8e5c85957098febcbcbaf09ab0b..189070eb2be3a7047cdfccfc8dd156a81b3b8dff 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -15,18 +15,14 @@ // specific language governing permissions and limitations // under the License. -#include "server/Config.h" - -#include #include -#include -#include #include #include #include #include -#include "config/ConfigMgr.h" +#include "config/YamlConfigMgr.h" +#include "server/Config.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" @@ -44,26 +40,24 @@ Config::GetInstance() { Status Config::LoadConfigFile(const std::string& filename) { if (filename.empty()) { - std::cerr << "ERROR: need specify config file" << std::endl; - exit(1); + return Status(SERVER_UNEXPECTED_ERROR, "No specified config file"); } - struct stat dirStat; - int statOK = stat(filename.c_str(), &dirStat); - if (statOK != 0) { - std::cerr << "ERROR: Config file not exist: " << filename << std::endl; - exit(1); + + struct stat file_stat; + if (stat(filename.c_str(), &file_stat) != 0) { + std::string str = "Config file not exist: " + filename; + return Status(SERVER_FILE_NOT_FOUND, str); } try { - ConfigMgr* mgr = const_cast(ConfigMgr::GetInstance()); - ErrorCode err = mgr->LoadConfigFile(filename); - if (err != 0) { - std::cerr << "Server failed to load config file: " << filename << std::endl; - exit(1); + ConfigMgr* mgr = YamlConfigMgr::GetInstance(); + Status s = mgr->LoadConfigFile(filename); + if (!s.ok()) { + return s; } } catch (YAML::Exception& e) { - std::cerr << "Server failed to load config file: " << filename << std::endl; - exit(1); + std::string str = "Exception occurs when loading config file: " + filename; + return Status(SERVER_UNEXPECTED_ERROR, str); } return Status::OK(); @@ -632,7 +626,7 @@ Config::CheckResourceConfigPool(const std::vector& value) { //////////////////////////////////////////////////////////////////////////////// ConfigNode& Config::GetConfigNode(const std::string& name) { - ConfigMgr* mgr = ConfigMgr::GetInstance(); + ConfigMgr* mgr = YamlConfigMgr::GetInstance(); ConfigNode& root_node = mgr->GetRootNode(); return root_node.GetChild(name); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 127bd9c05aef278da32690b09f8bd2362dfba1ee..9c754256a269e2a2c0a38c2ed586f53e263f0deb 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include @@ -110,13 +109,10 @@ class Config { private: ConfigNode& GetConfigNode(const std::string& name); - Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); - void PrintConfigSection(const std::string& config_node_name); diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index bb3bd012ab6d3da40471c4a862af85f70f5c0abe..a5b892ad47350a7b45d1c337500ca1fac84e4da3 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -15,19 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "server/DBWrapper.h" -#include "Config.h" -#include "db/DBFactory.h" -#include "utils/CommonUtil.h" -#include "utils/Log.h" -#include "utils/StringHelpFunctions.h" - #include #include #include #include #include +#include "db/DBFactory.h" +#include "server/Config.h" +#include "server/DBWrapper.h" +#include "utils/CommonUtil.h" +#include "utils/Log.h" +#include "utils/StringHelpFunctions.h" + namespace milvus { namespace server { @@ -35,9 +35,9 @@ Status DBWrapper::StartService() { Config& config = Config::GetInstance(); Status s; + // db config engine::DBOptions opt; - s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); if (!s.ok()) { return s; diff --git a/core/src/server/DBWrapper.h b/core/src/server/DBWrapper.h index 7016aa8805bf826678934bef880cd43590906301..0c74b3b9cb3ddbbc76770166d5ef72aef83aeb3d 100644 --- a/core/src/server/DBWrapper.h +++ b/core/src/server/DBWrapper.h @@ -17,12 +17,11 @@ #pragma once +#include + #include "db/DB.h" #include "utils/Status.h" -#include -#include - namespace milvus { namespace server { diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index d9aabfc1798c2894950cb2493ba75e78c8081cbc..465ed62ddfe4370b07e3598ad59be2f8d532912a 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -15,21 +15,15 @@ // specific language governing permissions and limitations // under the License. -#include "server/Server.h" - #include -#include -#include -#include -#include -//#include #include #include -#include "DBWrapper.h" #include "metrics/Metrics.h" #include "scheduler/SchedInst.h" #include "server/Config.h" +#include "server/DBWrapper.h" +#include "server/Server.h" #include "server/grpc_impl/GrpcServer.h" #include "utils/Log.h" #include "utils/LogUtil.h" @@ -143,7 +137,7 @@ Server::Daemonize() { } } -void +Status Server::Start() { if (daemonized_ != 0) { Daemonize(); @@ -151,18 +145,19 @@ Server::Start() { try { /* Read config file */ - if (LoadConfig() != SERVER_SUCCESS) { - std::cerr << "Milvus server fail to load config file" << std::endl; - return; + Status s = LoadConfig(); + if (!s.ok()) { + std::cerr << "ERROR: Milvus server fail to load config file" << std::endl; + return s; } /* log path is defined in Config file, so InitLog must be called after LoadConfig */ Config& config = Config::GetInstance(); std::string time_zone; - Status s = config.GetServerConfigTimeZone(time_zone); + s = config.GetServerConfigTimeZone(time_zone); if (!s.ok()) { std::cerr << "Fail to get server config timezone" << std::endl; - return; + return s; } if (time_zone.length() == 3) { @@ -179,8 +174,7 @@ Server::Start() { } if (setenv("TZ", time_zone.c_str(), 1) != 0) { - std::cerr << "Fail to setenv" << std::endl; - return; + return Status(SERVER_UNEXPECTED_ERROR, "Fail to setenv"); } tzset(); @@ -190,9 +184,10 @@ Server::Start() { server::SystemInfo::GetInstance().Init(); StartService(); - std::cout << "Milvus server start successfully." << std::endl; + return Status::OK(); } catch (std::exception& ex) { - std::cerr << "Milvus server encounter exception: " << ex.what(); + std::string str = "Milvus server encounter exception: " + std::string(ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, str); } } @@ -204,12 +199,12 @@ Server::Stop() { if (pid_fd_ != -1) { int ret = lockf(pid_fd_, F_ULOCK, 0); if (ret != 0) { - std::cerr << "Can't lock file: " << strerror(errno) << std::endl; + std::cerr << "ERROR: Can't lock file: " << strerror(errno) << std::endl; exit(0); } ret = close(pid_fd_); if (ret != 0) { - std::cerr << "Can't close file: " << strerror(errno) << std::endl; + std::cerr << "ERROR: Can't close file: " << strerror(errno) << std::endl; exit(0); } } @@ -218,31 +213,31 @@ Server::Stop() { if (!pid_filename_.empty()) { int ret = unlink(pid_filename_.c_str()); if (ret != 0) { - std::cerr << "Can't unlink file: " << strerror(errno) << std::endl; + std::cerr << "ERROR: Can't unlink file: " << strerror(errno) << std::endl; exit(0); } } StopService(); - std::cerr << "Milvus server is closed!" << std::endl; + std::cerr << "Milvus server exit..." << std::endl; } -ErrorCode +Status Server::LoadConfig() { Config& config = Config::GetInstance(); Status s = config.LoadConfigFile(config_filename_); if (!s.ok()) { - std::cerr << "Failed to load config file: " << config_filename_ << std::endl; - exit(0); + std::cerr << s.message() << std::endl; + return s; } s = config.ValidateConfig(); if (!s.ok()) { std::cerr << "Config check fail: " << s.message() << std::endl; - exit(0); + return s; } - return SERVER_SUCCESS; + return milvus::Status::OK(); } void diff --git a/core/src/server/Server.h b/core/src/server/Server.h index 658dbf7c3026cd432d98c46010cb8e364659d7a5..c0f2812f4c22a1e8febada67adfe5f2d78b69101 100644 --- a/core/src/server/Server.h +++ b/core/src/server/Server.h @@ -17,10 +17,8 @@ #pragma once -#include "utils/Status.h" - -#include #include +#include "utils/Status.h" namespace milvus { namespace server { @@ -34,7 +32,7 @@ class Server { Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, const std::string& log_config_file); - void + Status Start(); void Stop(); @@ -46,7 +44,7 @@ class Server { void Daemonize(); - ErrorCode + Status LoadConfig(); void