Merge branch 'branch-0.4.0' into 'branch-0.4.0'

merge 0.3.1

See merge request megasearch/milvus!398

Former-commit-id: 31d847e4f7c71f1034cd37e91980a6e72763ada8

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -1794,6 +1794,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
         return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what());
     }
 
+    //remove deleted table folder
+    //don't remove table folder until all its files has been deleted
     try {
         MetricCollector metric;
 

cpp/unittest/db/mem_test.cpp

@@ -449,3 +449,39 @@ TEST_F(DBTest, VECTOR_IDS_TEST)
     }
 }
 
+TEST_F(NewMemManagerTest, MEMMANAGER_TEST) {
+    int setenv_res = setenv("MILVUS_USE_OLD_MEM_MANAGER", "ON", 1);
+    ASSERT_TRUE(setenv_res == 0);
+
+    auto options = engine::OptionsFactory::Build();
+    options.meta.path = "/tmp/milvus_test";
+    options.meta.backend_uri = "sqlite://:@:/";
+    auto db_ = engine::DBFactory::Build(options);
+
+    engine::meta::TableSchema table_info = BuildTableSchema();
+    engine::Status stat = db_->CreateTable(table_info);
+
+    engine::meta::TableSchema table_info_get;
+    table_info_get.table_id_ = TABLE_NAME;
+    stat = db_->DescribeTable(table_info_get);
+    ASSERT_STATS(stat);
+    ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
+
+    auto start_time = METRICS_NOW_TIME;
+
+    int insert_loop = 20;
+    for (int i = 0; i < insert_loop; ++i) {
+        int64_t nb = 40960;
+        std::vector<float> xb;
+        BuildVectors(nb, xb);
+        engine::IDNumbers vector_ids;
+        engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
+        ASSERT_TRUE(status.ok());
+    }
+    auto end_time = METRICS_NOW_TIME;
+    auto total_time = METRICS_MICROSECONDS(start_time, end_time);
+    LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
+
+    delete db_;
+    boost::filesystem::remove_all(options.meta.path);
+}

cpp/unittest/db/scheduler_test.cpp

@@ -57,7 +57,6 @@ TEST(DBSchedulerTest, TASK_QUEUE_TEST) {
     ptr = queue.Back();
     ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad);
     load_task->Execute();
-
 }
 
 TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) {

cpp/unittest/server/util_test.cpp

@@ -178,12 +178,15 @@ TEST(UtilTest, VALIDATE_TABLENAME_TEST) {
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
-    table_name = "中文";
+    table_name = "_!@#!@";
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
+    table_name = "中文";
+    res = server::ValidationUtil::ValidateTableName(table_name);
+    ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 
-    table_name = std::string('a', 32768);
+    table_name = std::string(10000, 'a');
     res = server::ValidationUtil::ValidateTableName(table_name);
     ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME);
 }
@@ -204,6 +207,15 @@ TEST(UtilTest, VALIDATE_INDEXTYPE_TEST) {
     ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE);
 }
 
+TEST(ValidationUtilTest, ValidateGpuTest) {
+    ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS);
+    ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS);
+
+    size_t memory = 0;
+    ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS);
+    ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS);
+}
+
 TEST(UtilTest, TIMERECORDER_TEST) {
     for(int64_t log_level = 0; log_level <= 6; log_level++) {
         if(log_level == 5) {

cpp/unittest/utils/ValidationUtilTest.cpp

-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-#include <gtest/gtest.h>
-
-#include "utils/ValidationUtil.h"
-#include "utils/Error.h"
-#include "db/ExecutionEngine.h"
-
-#include <string>
-
-using namespace zilliz::milvus;
-using namespace zilliz::milvus::server;
-
-TEST(ValidationUtilTest, TableNameTest) {
-    std::string table_name = "Normal123_";
-    ServerError res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_SUCCESS);
-
-    table_name = "12sds";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-
-    table_name = "";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-
-    table_name = "_asdasd";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_SUCCESS);
-
-    table_name = "!@#!@";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-
-    table_name = "_!@#!@";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-
-    table_name = "中文";
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-
-
-    table_name = std::string(10000, 'a');
-    res = ValidationUtil::ValidateTableName(table_name);
-    ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
-}
-
-
-TEST(ValidationUtilTest, TableDimensionTest) {
-    ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
-    ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS);
-    ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS);
-}
-
-TEST(ValidationUtilTest, TableIndexTypeTest) {
-    ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
-    for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) {
-        ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS);
-    }
-    ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
-}
-
-TEST(ValidationUtilTest, ValidateGpuTest) {
-    ASSERT_EQ(ValidationUtil::ValidateGpuIndex(0), SERVER_SUCCESS);
-    ASSERT_NE(ValidationUtil::ValidateGpuIndex(100), SERVER_SUCCESS);
-
-    size_t memory = 0;
-    ASSERT_EQ(ValidationUtil::GetGpuMemory(0, memory), SERVER_SUCCESS);
-    ASSERT_NE(ValidationUtil::GetGpuMemory(100, memory), SERVER_SUCCESS);
-}