merge 0.3.1

Former-commit-id: 6fa5ce5365752369865360d715659c56fc367f89

cpp/CHANGELOG.md

@@ -61,6 +61,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-257 - Update bzip2 download url
 - MS-288 - Update compile scripts
 - MS-330 - Stability test failed caused by server core dumped
+- MS-347 - Build index hangs again
 
 ## Improvement
 - MS-156 - Add unittest for merge result functions

cpp/src/db/Utils.cpp

@@ -85,16 +85,20 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
     return Status::OK();
 }
 
-Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id) {
-    std::string db_path = options.path;
-    std::string table_path = db_path + TABLES_FOLDER + table_id;
-    boost::filesystem::remove_all(table_path);
-    ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
-
-    for(auto& path : options.slave_paths) {
-        table_path = path + TABLES_FOLDER + table_id;
-        boost::filesystem::remove_all(table_path);
-        ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
+Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
+    std::vector<std::string> paths = options.slave_paths;
+    paths.push_back(options.path);
+
+    for(auto& path : paths) {
+        std::string table_path = path + TABLES_FOLDER + table_id;
+        if(force) {
+            boost::filesystem::remove_all(table_path);
+            ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
+        } else if(boost::filesystem::exists(table_path) &&
+                  boost::filesystem::is_empty(table_path)) {
+            boost::filesystem::remove_all(table_path);
+            ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
+        }
     }
 
     return Status::OK();

cpp/src/db/Utils.h

@@ -19,7 +19,7 @@ namespace utils {
 long GetMicroSecTimeStamp();
 
 Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
-Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id);
+Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
 
 Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
 Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -1652,15 +1652,14 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
 }
 
 Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
-
-
     auto now = utils::GetMicroSecTimeStamp();
+    std::set<std::string> table_ids;
+
+    //remove to_delete files
     try {
         MetricCollector metric;
 
         {
-
-
             ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
 
             if (connectionPtr == nullptr) {
@@ -1700,6 +1699,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
                 ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_;
 
                 idsToDelete.emplace_back(std::to_string(table_file.id_));
+
+                table_ids.insert(table_file.table_id_);
             }
 
             if (!idsToDelete.empty()) {
@@ -1734,12 +1735,11 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
         return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
     }
 
+    //remove to_delete tables
     try {
         MetricCollector metric;
 
         {
-
-
             ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
 
             if (connectionPtr == nullptr) {
@@ -1765,7 +1765,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
                     std::string table_id;
                     resRow["table_id"].to_string(table_id);
 
-                    utils::DeleteTablePath(options_, table_id);
+                    utils::DeleteTablePath(options_, table_id, false);//only delete empty folder
 
                     idsToDeleteSS << "id = " << std::to_string(id) << " OR ";
                 }
@@ -1794,6 +1794,39 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
         return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
     }
 
+    //remove deleted table folder
+    //don't remove table folder until all its files has been deleted
+    try {
+        MetricCollector metric;
+
+        {
+            ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab);
+
+            if (connectionPtr == nullptr) {
+                return Status::Error("Failed to connect to database server");
+            }
+
+            for(auto& table_id : table_ids) {
+                Query cleanUpFilesWithTTLQuery = connectionPtr->query();
+                cleanUpFilesWithTTLQuery << "SELECT file_id " <<
+                                         "FROM TableFiles " <<
+                                         "WHERE table_id = " << table_id << ";";
+
+                ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
+
+                StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
+
+                if (res.empty()) {
+                    utils::DeleteTablePath(options_, table_id);
+                }
+            }
+        }
+    } catch (const Exception &er) {
+        // Catch-all for any other MySQL++ exceptions
+        ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL" << ": " << er.what();
+        return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", er.what());
+    }
+
     return Status::OK();
 }
 

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -1069,6 +1069,9 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
 
 Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
     auto now = utils::GetMicroSecTimeStamp();
+    std::set<std::string> table_ids;
+
+    //remove to_delete files
     try {
         MetricCollector metric;
 
@@ -1098,6 +1101,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
                 ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_;
                 ConnectorPtr->remove<TableFileSchema>(table_file.id_);
 
+                table_ids.insert(table_file.table_id_);
             }
             return true;
         });
@@ -1111,6 +1115,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
         return HandleException("Encounter exception when clean table files", e);
     }
 
+    //remove to_delete tables
     try {
         MetricCollector metric;
 
@@ -1123,7 +1128,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
 
         auto commited = ConnectorPtr->transaction([&]() mutable {
             for (auto &table : tables) {
-                utils::DeleteTablePath(options_, std::get<1>(table));
+                utils::DeleteTablePath(options_, std::get<1>(table), false);//only delete empty folder
                 ConnectorPtr->remove<TableSchema>(std::get<0>(table));
             }
 
@@ -1139,6 +1144,23 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
         return HandleException("Encounter exception when clean table files", e);
     }
 
+    //remove deleted table folder
+    //don't remove table folder until all its files has been deleted
+    try {
+        MetricCollector metric;
+
+        for(auto& table_id : table_ids) {
+            auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_),
+                                                 where(c(&TableFileSchema::table_id_) == table_id));
+            if(selected.size() == 0) {
+                utils::DeleteTablePath(options_, table_id);
+            }
+        }
+
+    } catch (std::exception &e) {
+        return HandleException("Encounter exception when delete table folder", e);
+    }
+
     return Status::OK();
 }
 

cpp/unittest/db/db_tests.cpp

@@ -93,6 +93,8 @@ TEST_F(DBTest, CONFIG_TEST) {
 
 
 TEST_F(DBTest, DB_TEST) {
+    db_->Open(GetOptions(), &db_);
+
     engine::meta::TableSchema table_info = BuildTableSchema();
     engine::Status stat = db_->CreateTable(table_info);
 
@@ -161,6 +163,11 @@ TEST_F(DBTest, DB_TEST) {
     }
 
     search.join();
+
+    uint64_t count;
+    stat = db_->GetTableRowCount(TABLE_NAME, count);
+    ASSERT_STATS(stat);
+    ASSERT_TRUE(count > 0);
 };
 
 TEST_F(DBTest, SEARCH_TEST) {

cpp/unittest/db/mem_test.cpp

@@ -445,4 +445,41 @@ TEST_F(DBTest, VECTOR_IDS_TEST)
     for (auto i = 0; i < nb; i++) {
         ASSERT_EQ(vector_ids[i], i + nb);
     }
-}
\ No newline at end of file
+}
+
+TEST_F(NewMemManagerTest, MEMMANAGER_TEST) {
+    int setenv_res = setenv("MILVUS_USE_OLD_MEM_MANAGER", "ON", 1);
+    ASSERT_TRUE(setenv_res == 0);
+
+    auto options = engine::OptionsFactory::Build();
+    options.meta.path = "/tmp/milvus_test";
+    options.meta.backend_uri = "sqlite://:@:/";
+    auto db_ = engine::DBFactory::Build(options);
+
+    engine::meta::TableSchema table_info = BuildTableSchema();
+    engine::Status stat = db_->CreateTable(table_info);
+
+    engine::meta::TableSchema table_info_get;
+    table_info_get.table_id_ = TABLE_NAME;
+    stat = db_->DescribeTable(table_info_get);
+    ASSERT_STATS(stat);
+    ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
+
+    auto start_time = METRICS_NOW_TIME;
+
+    int insert_loop = 20;
+    for (int i = 0; i < insert_loop; ++i) {
+        int64_t nb = 40960;
+        std::vector<float> xb;
+        BuildVectors(nb, xb);
+        engine::IDNumbers vector_ids;
+        engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
+        ASSERT_TRUE(status.ok());
+    }
+    auto end_time = METRICS_NOW_TIME;
+    auto total_time = METRICS_MICROSECONDS(start_time, end_time);
+    LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
+
+    delete db_;
+    boost::filesystem::remove_all(options.meta.path);
+}

cpp/unittest/db/mysql_meta_test.cpp

@@ -194,6 +194,13 @@ TEST_F(DISABLED_MySQLTest, ARCHIVE_TEST_DAYS) {
         i++;
     }
 
+    bool has;
+    status = impl.HasNonIndexFiles(table_id, has);
+    ASSERT_TRUE(status.ok());
+
+    status = impl.UpdateTableFilesToIndex(table_id);
+    ASSERT_TRUE(status.ok());
+
     status = impl.DropAll();
     ASSERT_TRUE(status.ok());
 }

cpp/unittest/db/scheduler_test.cpp

@@ -56,6 +56,7 @@ TEST(DBSchedulerTest, TASK_QUEUE_TEST) {
 
     ptr = queue.Back();
     ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad);
+    load_task->Execute();
 }
 
 TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) {

cpp/unittest/server/config_test.cpp

@@ -87,6 +87,9 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
     server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH);
     ASSERT_EQ(err, server::SERVER_SUCCESS);
 
+    err = server::ServerConfig::GetInstance().ValidateConfig();
+    ASSERT_EQ(err, server::SERVER_SUCCESS);
+
     server::ConfigNode node1 = config.GetConfig("server_config");
     server::ConfigNode& node2 = config.GetConfig("cache_config");
     node1.Combine(node2);

cpp/unittest/server/util_test.cpp

@@ -178,12 +178,15 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) {
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
-    table_name = "中文";
+    table_name = "_!@#!@";
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
+    table_name = "中文";
+    res = server::ValidationUtil::ValidateTableName(table_name);
+    ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
-    table_name = std::string('a', 32768);
+    table_name = std::string(10000, 'a');
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 }
@@ -204,3 +207,22 @@ TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) {
     ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE);
 }
 
+TEST(ValidationUtilTest, ValidateGpuTest) {
+    ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS);
+    ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS);
+
+    size_t memory = 0;
+    ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS);
+    ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS);
+}
+
+TEST(UtilTest, TIMERECORDER_TEST) {
+    for(int64_t log_level = 0; log_level <= 6; log_level++) {
+        if(log_level == 5) {
+            continue; //skip fatal
+        }
+        server::TimeRecorder rc("time", log_level);
+        rc.RecordSection("end");
+    }
+}
+