Merge remote-tracking branch 'main/branch-0.4.0' into branch-0.4.0

Former-commit-id: c2a42f8e0cd12ca351c024638c41d56e4f0f22b3

cpp/CHANGELOG.md

@@ -6,6 +6,7 @@ Please mark all change in change log and use the ticket from JIRA.
 
 ## Bug
 - MS-119 - The problem of combining the log files
+- MS-121 - The problem that user can't change the time zone
 - MS-411 - Fix metric unittest linking error
 - MS-412 - Fix gpu cache logical error
 - MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash

cpp/conf/server_config.template

@@ -3,6 +3,7 @@ server_config:
   port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
   gpu_index: 0                # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
   mode: single                # milvus deployment type: single, cluster, read_only
+  time_zone: UTC+8            # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
 
 db_config:
   db_path: @MILVUS_DB_PATH@             # milvus data storage path

cpp/src/core/include/knowhere/index/preprocessor/normalize.h

-#pragma once
-
-#include <memory>
-#include "preprocessor.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-class NormalizePreprocessor : public Preprocessor {
- public:
-    DatasetPtr
-    Preprocess(const DatasetPtr &input) override;
-
- private:
-
-    void
-    Normalize(float *arr, int64_t dimension);
-};
-
-
-using NormalizePreprocessorPtr = std::shared_ptr<NormalizePreprocessor>;
-
-
-} // namespace knowhere
-} // namespace zilliz
+//#pragma once
+//
+//#include <memory>
+//#include "preprocessor.h"
+//
+//
+//namespace zilliz {
+//namespace knowhere {
+//
+//class NormalizePreprocessor : public Preprocessor {
+// public:
+//    DatasetPtr
+//    Preprocess(const DatasetPtr &input) override;
+//
+// private:
+//
+//    void
+//    Normalize(float *arr, int64_t dimension);
+//};
+//
+//
+//using NormalizePreprocessorPtr = std::shared_ptr<NormalizePreprocessor>;
+//
+//
+//} // namespace knowhere
+//} // namespace zilliz

cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h

@@ -27,8 +27,8 @@ class CPUKDTRNG : public VectorIndex {
     Load(const BinarySet &index_array) override;
 
  public:
-    PreprocessorPtr
-    BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override;
+    //PreprocessorPtr
+    //BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override;
     int64_t Count() override;
     int64_t Dimension() override;
 

cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp

-
-#include "knowhere/index/vector_index/definitions.h"
-#include "knowhere/common/config.h"
-#include "knowhere/index/preprocessor/normalize.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-DatasetPtr
-NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) {
+//
+//#include "knowhere/index/vector_index/definitions.h"
+//#include "knowhere/common/config.h"
+//#include "knowhere/index/preprocessor/normalize.h"
+//
+//
+//namespace zilliz {
+//namespace knowhere {
+//
+//DatasetPtr
+//NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) {
 //    // TODO: wrap dataset->tensor
 //    auto tensor = dataset->tensor()[0];
 //    auto p_data = (float *)tensor->raw_mutable_data();
@@ -19,24 +19,24 @@ NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) {
 //    for (auto i = 0; i < rows; ++i) {
 //        Normalize(&(p_data[i * dimension]), dimension);
 //    }
-}
-
-void
-NormalizePreprocessor::Normalize(float *arr, int64_t dimension) {
-    //double vector_length = 0;
-    //for (auto j = 0; j < dimension; j++) {
-    //    double val = arr[j];
-    //    vector_length += val * val;
-    //}
-    //vector_length = std::sqrt(vector_length);
-    //if (vector_length < 1e-6) {
-    //    auto val = (float) (1.0 / std::sqrt((double) dimension));
-    //    for (int j = 0; j < dimension; j++) arr[j] = val;
-    //} else {
-    //    for (int j = 0; j < dimension; j++) arr[j] = (float) (arr[j] / vector_length);
-    //}
-}
-
-} // namespace knowhere
-} // namespace zilliz
+//}
+//
+//void
+//NormalizePreprocessor::Normalize(float *arr, int64_t dimension) {
+//    double vector_length = 0;
+//    for (auto j = 0; j < dimension; j++) {
+//        double val = arr[j];
+//        vector_length += val * val;
+//    }
+//    vector_length = std::sqrt(vector_length);
+//    if (vector_length < 1e-6) {
+//        auto val = (float) (1.0 / std::sqrt((double) dimension));
+//        for (int j = 0; j < dimension; j++) arr[j] = val;
+//    } else {
+//        for (int j = 0; j < dimension; j++) arr[j] = (float) (arr[j] / vector_length);
+//    }
+//}
+//
+//} // namespace knowhere
+//} // namespace zilliz
 

cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp

@@ -9,7 +9,7 @@
 
 #include "knowhere/index/vector_index/cpu_kdt_rng.h"
 #include "knowhere/index/vector_index/definitions.h"
-#include "knowhere/index/preprocessor/normalize.h"
+//#include "knowhere/index/preprocessor/normalize.h"
 #include "knowhere/index/vector_index/kdt_parameters.h"
 #include "knowhere/adapter/sptag.h"
 #include "knowhere/common/exception.h"
@@ -60,10 +60,10 @@ CPUKDTRNG::Load(const BinarySet &binary_set) {
     index_ptr_->LoadIndexFromMemory(index_blobs);
 }
 
-PreprocessorPtr
-CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) {
-    return std::make_shared<NormalizePreprocessor>();
-}
+//PreprocessorPtr
+//CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) {
+//    return std::make_shared<NormalizePreprocessor>();
+//}
 
 IndexModelPtr
 CPUKDTRNG::Train(const DatasetPtr &origin, const Config &train_config) {
@@ -72,7 +72,7 @@ CPUKDTRNG::Train(const DatasetPtr &origin, const Config &train_config) {
 
     //if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine
     //    && preprocessor_) {
-        preprocessor_->Preprocess(dataset);
+    //    preprocessor_->Preprocess(dataset);
     //}
 
     auto vectorset = ConvertToVectorSet(dataset);
@@ -90,7 +90,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) {
 
     //if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine
     //    && preprocessor_) {
-        preprocessor_->Preprocess(dataset);
+    //    preprocessor_->Preprocess(dataset);
     //}
 
     auto vectorset = ConvertToVectorSet(dataset);

cpp/src/core/test/test_kdt.cpp

@@ -8,6 +8,7 @@
 
 #include <iostream>
 #include <sstream>
+#include "knowhere/common/exception.h"
 
 #include "knowhere/index/vector_index/cpu_kdt_rng.h"
 #include "knowhere/index/vector_index/definitions.h"
@@ -125,6 +126,10 @@ TEST_P(KDTTest, kdt_serialize) {
     auto result = new_index->Search(query_dataset, search_cfg);
     AssertAnns(result, nq, k);
     PrintResult(result, nq, k);
+    ASSERT_EQ(new_index->Count(), nb);
+    ASSERT_EQ(new_index->Dimension(), dim);
+    ASSERT_THROW({new_index->Clone();}, zilliz::knowhere::KnowhereException);
+    ASSERT_NO_THROW({new_index->Seal();});
 
     {
         int fileno = 0;

cpp/src/main.cpp

@@ -16,7 +16,6 @@
 
 #include "utils/SignalUtil.h"
 #include "utils/CommonUtil.h"
-#include "utils/LogUtil.h"
 
 INITIALIZE_EASYLOGGINGPP
 
@@ -98,10 +97,8 @@ main(int argc, char *argv[]) {
         }
     }
 
-    zilliz::milvus::server::InitLog(log_config_file);
-
     server::Server* server_ptr = server::Server::Instance();
-    server_ptr->Init(start_daemonized, pid_filename, config_filename);
+    server_ptr->Init(start_daemonized, pid_filename, config_filename, log_config_file);
     return server_ptr->Start();
 }
 

cpp/src/server/Server.cpp

@@ -7,6 +7,7 @@
 #include "Server.h"
 #include "server/grpc_impl/GrpcMilvusServer.h"
 #include "utils/Log.h"
+#include "utils/LogUtil.h"
 #include "utils/SignalUtil.h"
 #include "utils/TimeRecorder.h"
 #include "metrics/Metrics.h"
@@ -24,11 +25,12 @@
 #include "metrics/Metrics.h"
 #include "DBWrapper.h"
 
+
 namespace zilliz {
 namespace milvus {
 namespace server {
 
-Server*
+Server *
 Server::Instance() {
     static Server server;
     return &server;
@@ -42,10 +44,14 @@ Server::~Server() {
 }
 
 void
-Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename) {
+Server::Init(int64_t daemonized,
+             const std::string &pid_filename,
+             const std::string &config_filename,
+             const std::string &log_config_file) {
     daemonized_ = daemonized;
     pid_filename_ = pid_filename;
     config_filename_ = config_filename;
+    log_config_file_ = log_config_file;
 }
 
 void
@@ -54,7 +60,7 @@ Server::Daemonize() {
         return;
     }
 
-    SERVER_LOG_INFO << "Milvus server run in daemonize mode";
+    std::cout << "Milvus server run in daemonize mode";
 
 //    std::string log_path(GetLogDirFullPath());
 //    log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)";
@@ -101,7 +107,7 @@ Server::Daemonize() {
 
     // Change the working directory to root
     int ret = chdir("/");
-    if(ret != 0){
+    if (ret != 0) {
         return;
     }
 
@@ -110,7 +116,7 @@ Server::Daemonize() {
         close(fd);
     }
 
-    SERVER_LOG_INFO << "Redirect stdin/stdout/stderr to /dev/null";
+    std::cout << "Redirect stdin/stdout/stderr to /dev/null";
 
     // Redirect stdin/stdout/stderr to /dev/null
     stdin = fopen("/dev/null", "r");
@@ -120,17 +126,17 @@ Server::Daemonize() {
     if (!pid_filename_.empty()) {
         pid_fd = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640);
         if (pid_fd < 0) {
-            SERVER_LOG_INFO << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno);
+            std::cout << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno);
             exit(EXIT_FAILURE);
         }
         if (lockf(pid_fd, F_TLOCK, 0) < 0) {
-            SERVER_LOG_INFO << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno);
+            std::cout << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno);
             exit(EXIT_FAILURE);
         }
 
         std::string pid_file_context = std::to_string(getpid());
         ssize_t res = write(pid_fd, pid_file_context.c_str(), pid_file_context.size());
-        if(res != 0){
+        if (res != 0) {
             return;
         }
     }
@@ -146,7 +152,7 @@ Server::Start() {
     do {
         try {
             // Read config file
-            if(LoadConfig() != SERVER_SUCCESS) {
+            if (LoadConfig() != SERVER_SUCCESS) {
                 return 1;
             }
 
@@ -154,6 +160,27 @@ Server::Start() {
             ServerConfig &config = ServerConfig::GetInstance();
             ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
 
+            std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8");
+            if (time_zone.length() == 3) {
+                time_zone = "CUT";
+            } else {
+                int time_bias = std::stoi(time_zone.substr(3, std::string::npos));
+                if (time_bias == 0)
+                    time_zone = "CUT";
+                else if (time_bias > 0) {
+                    time_zone = "CUT" + std::to_string(-time_bias);
+                } else {
+                    time_zone = "CUT+" + std::to_string(-time_bias);
+                }
+            }
+
+            if (setenv("TZ", time_zone.c_str(), 1) != 0) {
+                return -1;
+            }
+            tzset();
+
+            InitLog(log_config_file_);
+
             // Handle Signal
             signal(SIGINT, SignalUtil::HandleSignal);
             signal(SIGHUP, SignalUtil::HandleSignal);
@@ -164,12 +191,12 @@ Server::Start() {
             std::cout << "Milvus server start successfully." << std::endl;
             StartService();
 
-        } catch(std::exception& ex){
-            SERVER_LOG_ERROR << "Milvus server encounter exception: " << std::string(ex.what())
-                             << "Is another server instance running?";
+        } catch (std::exception &ex) {
+            std::cerr << "Milvus server encounter exception: " << std::string(ex.what())
+                      << "Is another server instance running?";
             break;
         }
-    } while(false);
+    } while (false);
 
     Stop();
     return 0;
@@ -182,12 +209,12 @@ Server::Stop() {
     // Unlock and close lockfile
     if (pid_fd != -1) {
         int ret = lockf(pid_fd, F_ULOCK, 0);
-        if(ret != 0){
+        if (ret != 0) {
             std::cout << "Can't lock file: " << strerror(errno) << std::endl;
             exit(0);
         }
         ret = close(pid_fd);
-        if(ret != 0){
+        if (ret != 0) {
             std::cout << "Can't close file: " << strerror(errno) << std::endl;
             exit(0);
         }
@@ -196,7 +223,7 @@ Server::Stop() {
     // Try to delete lockfile
     if (!pid_filename_.empty()) {
         int ret = unlink(pid_filename_.c_str());
-        if(ret != 0){
+        if (ret != 0) {
             std::cout << "Can't unlink file: " << strerror(errno) << std::endl;
             exit(0);
         }
@@ -214,7 +241,7 @@ ErrorCode
 Server::LoadConfig() {
     ServerConfig::GetInstance().LoadConfigFile(config_filename_);
     ErrorCode err = ServerConfig::GetInstance().ValidateConfig();
-    if(err != SERVER_SUCCESS){
+    if (err != SERVER_SUCCESS) {
         exit(0);
     }
 

cpp/src/server/Server.h

@@ -18,7 +18,7 @@ class Server {
    public:
     static Server* Instance();
 
-    void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename);
+    void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, const std::string &log_config_file);
     int Start();
     void Stop();
 
@@ -40,6 +40,7 @@ class Server {
     int pid_fd = -1;
     std::string pid_filename_;
     std::string config_filename_;
+    std::string log_config_file_;
 };  // Server
 
 }   // server

cpp/src/server/ServerConfig.cpp

@@ -105,8 +105,7 @@ ServerConfig::CheckServerConfig() {
     if (ValidationUtil::ValidateStringIsNumber(port_str) != SERVER_SUCCESS) {
         std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl;
         okay = false;
-    }
-    else {
+    } else {
         int32_t port = std::stol(port_str);
         if (port < 1025 | port > 65534) {
             std::cerr << "ERROR: port " << port_str << " out of range [1025, 65534]" << std::endl;
@@ -118,8 +117,7 @@ ServerConfig::CheckServerConfig() {
     if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) {
         std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl;
         okay = false;
-    }
-    else {
+    } else {
         int32_t gpu_index = std::stol(gpu_index_str);
         if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
             std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl;
@@ -133,6 +131,25 @@ ServerConfig::CheckServerConfig() {
         okay = false;
     }
 
+    std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8");
+    int flag = 0;
+    if(time_zone.length() < 3)
+        flag = 1;
+    else if(time_zone.substr(0, 3) != "UTC")
+        flag = 1;
+    else if(time_zone.length() > 3){
+        try {
+            stoi(time_zone.substr(3, std::string::npos));
+        }
+        catch (std::invalid_argument &) {
+            flag = 1;
+        }
+    }
+    if(flag == 1){
+        std::cerr << "ERROR: time_zone " << time_zone << " is not in a right format" << std::endl;
+        okay = false;
+    }
+
     return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT);
 }
 
@@ -359,8 +376,7 @@ ServerConfig::CheckEngineConfig() {
     if (ValidationUtil::ValidateStringIsNumber(omp_thread_num_str) != SERVER_SUCCESS) {
         std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl;
         okay = false;
-    }
-    else {
+    } else {
         int32_t omp_thread = std::stol(omp_thread_num_str);
         uint32_t sys_thread_cnt = 8;
         if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
@@ -448,8 +464,7 @@ ServerConfig::CheckResourceConfig() {
         if (ValidationUtil::ValidateStringIsNumber(device_id_str) != SERVER_SUCCESS) {
             std::cerr << "ERROR: device_id " << device_id_str << " is not a number" << std::endl;
             okay = false;
-        }
-        else {
+        } else {
             device_id = std::stol(device_id_str);
         }
 
@@ -461,8 +476,7 @@ ServerConfig::CheckResourceConfig() {
 
         if (type == "DISK") {
             hasDisk = true;
-        }
-        else if (type == "CPU") {
+        } else if (type == "CPU") {
             hasCPU = true;
             if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) {
                 hasExecutor = true;
@@ -541,8 +555,7 @@ ServerConfig::CheckResourceConfig() {
         if (delimiter_pos == std::string::npos) {
             std::cerr << "ERROR: invalid endpoint format: " << endpoint_str << std::endl;
             okay = false;
-        }
-        else {
+        } else {
             std::string left_resource = endpoint_str.substr(0, delimiter_pos);
             if (resource_list.find(left_resource) == resource_list.end()) {
                 std::cerr << "ERROR: left resource " << left_resource << " does not exist" << std::endl;

cpp/src/server/ServerConfig.h

@@ -19,6 +19,7 @@ static const char* CONFIG_SERVER_ADDRESS = "address";
 static const char* CONFIG_SERVER_PORT = "port";
 static const char* CONFIG_CLUSTER_MODE = "mode";
 static const char* CONFIG_GPU_INDEX = "gpu_index";
+static const char* CONFIG_TIME_ZONE = "time_zone";
 
 static const char* CONFIG_DB = "db_config";
 static const char* CONFIG_DB_URL = "db_backend_url";