MS-578 makesure milvus5.0 dont crack 0.3.1 data

Former-commit-id: 9eac427b8aa881afe2bf4e2fd6964690a5b553d1

cpp/CHANGELOG.md

@@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-562 - Add JobMgr and TaskCreator in Scheduler
 - MS-566 - Refactor cmake
 - MS-555 - Remove old scheduler
+- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data
 
 ## New Feature
 

cpp/src/db/Options.cpp

@@ -17,7 +17,7 @@
 
 #include "Options.h"
 #include "utils/Exception.h"
-#include "utils/easylogging++.h"
+#include "utils/Log.h"
 
 #include <stdlib.h>
 #include <assert.h>
@@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
         std::vector<std::string> kv;
         boost::algorithm::split(kv, token, boost::is_any_of(":"));
         if (kv.size() != 2) {
-            LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
+            ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
             continue;
         }
         if (kv[0] != "disk" && kv[0] != "days") {
-            LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
+            ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
             continue;
         }
         try {
@@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
             criterias_[kv[0]] = value;
         }
         catch (std::out_of_range&){
-            LOG(ERROR) << "Out of range: '" << kv[1] << "'";
-            throw OutOfRangeException();
+            std::string msg = "Out of range: '" + kv[1] + "'";
+            ENGINE_LOG_ERROR << msg;
+            throw InvalidArgumentException(msg);
         }
         catch (...){
-            LOG(ERROR) << "Invalid argument: '" << kv[1] << "'";
-            throw InvalidArgumentException();
+            std::string msg = "Invalid argument: '" + kv[1] + "'";
+            ENGINE_LOG_ERROR << msg;
+            throw InvalidArgumentException(msg);
         }
     }
 }
 
 void ArchiveConf::ParseType(const std::string& type) {
     if (type != "delete" && type != "swap") {
-        LOG(ERROR) << "Invalid argument: type='" << type << "'";
-        throw InvalidArgumentException();
+        std::string msg = "Invalid argument: type='" + type + "'";
+        throw InvalidArgumentException(msg);
     }
     type_ = type;
 }

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
       nlist_(nlist) {
 
     index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
-    if (!index_) throw Exception("Create Empty VecIndex");
+    if (!index_) {
+        throw Exception(DB_ERROR, "Could not create VecIndex");
+    }
 
     Config build_cfg;
     build_cfg["dim"] = dimension;
     build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
     AutoGenParams(index_->GetType(), 0, build_cfg);
     auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
-    if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
+    if (ec != KNOWHERE_SUCCESS) {
+        throw Exception(DB_ERROR, "Build index error");
+    }
 }
 
 ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
@@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
 
     auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
-        throw Exception("Create Empty VecIndex");
+        throw Exception(DB_ERROR, "Could not create VecIndex");
     }
 
     Config build_cfg;
@@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
                                  from_index->GetRawVectors(),
                                  from_index->GetRawIds(),
                                  build_cfg);
-    if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
+    if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); }
 
     return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
 }

cpp/src/db/meta/Meta.h

@@ -31,6 +31,9 @@ namespace milvus {
 namespace engine {
 namespace meta {
 
+static const char* META_TABLES = "Tables";
+static const char* META_TABLEFILES = "TableFiles";
+
 class Meta {
  public:
     virtual ~Meta() = default;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -19,6 +19,7 @@
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
 #include "utils/Log.h"
+#include "utils/Exception.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 
@@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) {
 
 inline auto StoragePrototype(const std::string &path) {
     return make_storage(path,
-                        make_table("Tables",
+                        make_table(META_TABLES,
                                    make_column("id", &TableSchema::id_, primary_key()),
                                    make_column("table_id", &TableSchema::table_id_, unique()),
                                    make_column("state", &TableSchema::state_),
@@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) {
                                    make_column("engine_type", &TableSchema::engine_type_),
                                    make_column("nlist", &TableSchema::nlist_),
                                    make_column("metric_type", &TableSchema::metric_type_)),
-                        make_table("TableFiles",
+                        make_table(META_TABLEFILES,
                                    make_column("id", &TableFileSchema::id_, primary_key()),
                                    make_column("table_id", &TableFileSchema::table_id_),
                                    make_column("engine_type", &TableFileSchema::engine_type_),
@@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() {
 
     ConnectorPtr = std::make_unique<ConnectorT>(StoragePrototype(options_.path + "/meta.sqlite"));
 
+    //old meta could be recreated since schema changed, throw exception if meta schema is not compatible
+    auto ret = ConnectorPtr->sync_schema_simulate();
+    if(ret.find(META_TABLES) != ret.end()
+        && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
+            throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+    }
+    if(ret.find(META_TABLEFILES) != ret.end()
+       && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) {
+        throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+    }
+
     ConnectorPtr->sync_schema();
     ConnectorPtr->open_forever(); // thread safe option
     ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
@@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() {
     ENGINE_LOG_DEBUG << "Drop all sqlite meta";
 
     try {
-        ConnectorPtr->drop_table("Tables");
-        ConnectorPtr->drop_table("TableFiles");
+        ConnectorPtr->drop_table(META_TABLES);
+        ConnectorPtr->drop_table(META_TABLEFILES);
     } catch (std::exception &e) {
         return HandleException("Encounter exception when drop all meta", e.what());
     }

cpp/src/main.cpp

@@ -102,10 +102,6 @@ main(int argc, char *argv[]) {
         }
     }
 
-    server::Server &server = server::Server::Instance();
-    server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
-    server.Start();
-
     /* Handle Signal */
     signal(SIGHUP, server::SignalUtil::HandleSignal);
     signal(SIGINT, server::SignalUtil::HandleSignal);
@@ -114,6 +110,10 @@ main(int argc, char *argv[]) {
     signal(SIGUSR2, server::SignalUtil::HandleSignal);
     signal(SIGTERM, server::SignalUtil::HandleSignal);
 
+    server::Server &server = server::Server::Instance();
+    server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
+    server.Start();
+
     /* wait signal */
     pause();
 

cpp/src/server/DBWrapper.cpp

@@ -109,15 +109,10 @@ Status DBWrapper::StartService() {
     }
 
     //create db instance
-    std::string msg = opt.meta.path;
     try {
         db_ = engine::DBFactory::Build(opt);
     } catch(std::exception& ex) {
-        msg = ex.what();
-    }
-
-    if(db_ == nullptr) {
-        std::cerr << "ERROR! Failed to open database: " << msg << std::endl;
+        std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl;
         kill(0, SIGUSR1);
     }
 

cpp/src/server/Server.cpp

@@ -196,8 +196,8 @@ Server::Start() {
         server::Metrics::GetInstance().Init();
         server::SystemInfo::GetInstance().Init();
 
-        std::cout << "Milvus server start successfully." << std::endl;
         StartService();
+        std::cout << "Milvus server start successfully." << std::endl;
 
     } catch (std::exception &ex) {
         std::cerr << "Milvus server encounter exception: " << ex.what();

cpp/src/utils/Error.h

@@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
 constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
 constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
 constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
+constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
 
 //knowhere error code
 constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);

cpp/src/utils/Exception.h

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "utils/Error.h"
+
 #include <exception>
 #include <string>
 
@@ -25,13 +27,14 @@ namespace milvus {
 
 class Exception : public std::exception {
 public:
-    Exception(const std::string& message)
-        : message_(message) {
+    Exception(ErrorCode code, const std::string& message)
+        : code_(code_),
+          message_(message) {
         }
 
-    Exception()
-        : message_() {
-        }
+    ErrorCode code() const throw() {
+        return code_;
+    }
 
     virtual const char* what() const throw() {
         if (message_.empty()) {
@@ -44,20 +47,21 @@ public:
     virtual ~Exception() throw() {};
 
 protected:
-
+    ErrorCode code_;
     std::string message_;
 };
 
 class InvalidArgumentException : public Exception {
 public:
-    InvalidArgumentException() : Exception("Invalid Argument"){};
-    InvalidArgumentException(const std::string& message) : Exception(message) {};
-};
+    InvalidArgumentException()
+        : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
+
+    };
+    InvalidArgumentException(const std::string& message)
+        : Exception(SERVER_INVALID_ARGUMENT, message) {
+
+    };
 
-class OutOfRangeException : public Exception {
-public:
-    OutOfRangeException() : Exception("Out Of Range"){};
-    OutOfRangeException(const std::string& message) : Exception(message) {};
 };
 
 } // namespace milvus

cpp/src/utils/Log.h

@@ -24,7 +24,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define SERVER_DOMAIN_NAME "[SERVER] "
-#define SERVER_ERROR_TEXT "SERVER Error:"
 
 #define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME
 #define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME
@@ -35,7 +34,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define ENGINE_DOMAIN_NAME "[ENGINE] "
-#define ENGINE_ERROR_TEXT "ENGINE Error:"
 
 #define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
 #define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
@@ -46,7 +44,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define WRAPPER_DOMAIN_NAME "[WRAPPER] "
-#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
 
 #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
 #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME

cpp/src/utils/LogUtil.cpp

@@ -17,10 +17,8 @@
 
 #include "LogUtil.h"
 #include "server/ServerConfig.h"
-#include "easylogging++.h"
 
 #include <ctype.h>
-
 #include <string>
 #include <libgen.h>
 

cpp/unittest/db/db_tests.cpp

@@ -24,10 +24,7 @@
 #include "utils/CommonUtil.h"
 
 #include <gtest/gtest.h>
-#include "utils/easylogging++.h"
-
 #include <boost/filesystem.hpp>
-
 #include <thread>
 #include <random>
 

cpp/unittest/db/meta_tests.cpp

@@ -15,17 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <stdlib.h>
-#include <time.h>
-
 #include "utils.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/Utils.h"
 #include "db/meta/MetaConsts.h"
 
+#include <gtest/gtest.h>
+#include <thread>
+#include <stdlib.h>
+#include <time.h>
+
 using namespace zilliz::milvus;
 using namespace zilliz::milvus::engine;
 

cpp/unittest/db/misc_test.cpp

@@ -21,7 +21,6 @@
 #include "db/Utils.h"
 #include "utils/Status.h"
 #include "utils/Exception.h"
-#include "utils/easylogging++.h"
 
 #include <gtest/gtest.h>
 #include <thread>
@@ -31,13 +30,13 @@
 using namespace zilliz::milvus;
 
 TEST(DBMiscTest, EXCEPTION_TEST) {
-    Exception ex1("");
+    Exception ex1(100, "error");
     std::string what = ex1.what();
-    ASSERT_FALSE(what.empty());
+    ASSERT_EQ(what, "error");
+    ASSERT_EQ(ex1.code(), 100);
 
-    OutOfRangeException ex2;
-    what = ex2.what();
-    ASSERT_FALSE(what.empty());
+    InvalidArgumentException ex2;
+    ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT);
 }
 
 TEST(DBMiscTest, OPTIONS_TEST) {

cpp/unittest/db/mysql_db_test.cpp

@@ -21,7 +21,6 @@
 #include "db/meta/MetaConsts.h"
 
 #include <gtest/gtest.h>
-#include "utils/easylogging++.h"
 #include <boost/filesystem.hpp>
 
 #include <thread>

cpp/unittest/db/mysql_meta_test.cpp

@@ -15,20 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <stdlib.h>
-#include <time.h>
-
 #include "utils.h"
 #include "db/meta/MySQLMetaImpl.h"
 #include "db/Utils.h"
 #include "db/meta/MetaConsts.h"
 
-#include "mysql++/mysql++.h"
-
 #include <iostream>
+#include <thread>
+#include <stdlib.h>
+#include <time.h>
+#include <gtest/gtest.h>
+#include <mysql++/mysql++.h>
 
 using namespace zilliz::milvus;
 using namespace zilliz::milvus::engine;

cpp/unittest/main.cpp

@@ -15,13 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <gmock/gmock.h>
 #include "utils/easylogging++.h"
-
 #include "server/ServerConfig.h"
 #include "utils/CommonUtil.h"
 
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
 INITIALIZE_EASYLOGGINGPP
 
 using namespace zilliz::milvus;

cpp/unittest/server/util_test.cpp

@@ -15,14 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <boost/filesystem.hpp>
-#include <src/utils/SignalUtil.h>
-
+#include "utils/SignalUtil.h"
 #include "utils/CommonUtil.h"
 #include "utils/Error.h"
 #include "utils/StringHelpFunctions.h"
@@ -32,6 +25,12 @@
 #include "utils/ValidationUtil.h"
 #include "db/engine/ExecutionEngine.h"
 
+#include <thread>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <boost/filesystem.hpp>
+#include <gtest/gtest.h>
+
 using namespace zilliz::milvus;
 
 namespace {

cpp/unittest/wrapper/wrapper_test.cpp

@@ -16,14 +16,13 @@
 // under the License.
 
 
-#include <gtest/gtest.h>
 #include "utils/easylogging++.h"
-
-#include "src/wrapper/vec_index.h"
+#include "wrapper/vec_index.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
-
 #include "utils.h"
 
+#include <gtest/gtest.h>
+
 INITIALIZE_EASYLOGGINGPP
 
 using namespace zilliz::milvus::engine;