Merge branch 'revert-5f523800' into 'branch-0.3.0'

Revert "Merge branch 'mysql-0.3.0' into 'branch-0.3.0'"

See merge request megasearch/vecwise_engine!144

Former-commit-id: 2fec20a2409c67e50c9081d1e3626d273e20489e

cpp/conf/server_config.template

@@ -15,7 +15,6 @@ db_config:
   index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
   archive_disk_threshold: 512      # triger archive action if storage size exceed this value, unit: GB
   archive_days_threshold: 30       # files older than x days will be archived, unit: day
-  sql_echo: on                     # print sql statement in debug log
 
 metric_config:
   is_startup: off      # if monitoring start: on, off

cpp/coverage.sh

 #!/bin/bash
 
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib
+
 LCOV_CMD="lcov"
 LCOV_GEN_CMD="genhtml"
 
@@ -12,26 +14,6 @@ DIR_LCOV_OUTPUT="lcov_out"
 DIR_GCNO="cmake_build"
 DIR_UNITTEST="milvus/bin"
 
-MYSQL_USER_NAME=root
-MYSQL_PASSWORD=Fantast1c
-MYSQL_HOST='192.168.1.194'
-MYSQL_PORT='3306'
-
-MYSQL_DB_NAME=`date +%s%N`
-
-function mysql_exc()
-{
-    cmd=$1
-    mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}"
-    if [ $? -ne 0 ]; then
-        echo "mysql $cmd run failed"
-    fi
-}
-
-mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};"
-mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';"
-mysql_exc "FLUSH PRIVILEGES;"
-
 # get baseline
 ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}"
 if [ $? -ne 0 ]; then
@@ -40,13 +22,14 @@ if [ $? -ne 0 ]; then
 fi
 
 for test in `ls ${DIR_UNITTEST}`; do
-    echo $test
     case ${test} in
         db_test)
             # set run args for db_test
-            args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}"
+            args="mysql://root:Fantast1c@192.168.1.194:3306/test"
+            ;;
         *_test)
             args=""
+            ;;
     esac
     # run unittest
     ./${DIR_UNITTEST}/${test} "${args}"
@@ -55,8 +38,6 @@ for test in `ls ${DIR_UNITTEST}`; do
     fi
 done
 
-mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};"
-
 # gen test converage
 ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c
 # merge coverage
@@ -69,4 +50,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
     "*/cmake_build/*_ep-prefix/*" \
 
 # gen html report
-${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/
\ No newline at end of file
+${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/

cpp/src/db/MySQLConnectionPool.h

@@ -69,10 +69,6 @@ public:
         max_idle_time_ = max_idle;
     }
 
-    std::string getDB() {
-        return db_;
-    }
-
 protected:
 
     // Superclass overrides

cpp/src/db/MySQLMetaImpl.cpp

@@ -1733,30 +1733,17 @@ namespace meta {
 //                ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use  = " << mysql_connection_pool_->getConnectionsInUse();
 //            }
 
+            ENGINE_LOG_DEBUG << "Remove table file type as NEW";
             Query cleanUpQuery = connectionPtr->query();
-            cleanUpQuery << "SELECT table_name " <<
-                            "FROM information_schema.tables " <<
-                            "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " <<
-                            "AND table_name = " << quote << "TableFiles" << quote << ";";
+            cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
 
             if (options_.sql_echo) {
                 ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
             }
 
-            StoreQueryResult res = cleanUpQuery.store();
-            assert(res);
-            if (!res.empty()) {
-                ENGINE_LOG_DEBUG << "Remove table file type as NEW";
-                cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
-
-                if (options_.sql_echo) {
-                    ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
-                }
-
-                if (!cleanUpQuery.exec()) {
-                    ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
-                    return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
-                }
+            if (!cleanUpQuery.exec()) {
+                ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
+                return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
             }
 
         } catch (const BadQuery& er) {

cpp/unittest/db/mysql_meta_test.cpp

@@ -21,8 +21,21 @@
 
 using namespace zilliz::milvus::engine;
 
-TEST_F(MySQLTest, TABLE_TEST) {
+//TEST_F(MySQLTest, InitializeTest) {
+//    DBMetaOptions options;
+//    //dialect+driver://username:password@host:port/database
+//    options.backend_uri = "mysql://root:1234@:/test";
+//    meta::MySQLMetaImpl impl(options);
+//    auto status = impl.Initialize();
+//    std::cout << status.ToString() << std::endl;
+//    ASSERT_TRUE(status.ok());
+//}
+
+TEST_F(MySQLTest, core) {
     DBMetaOptions options;
+//    //dialect+driver://username:password@host:port/database
+//    options.backend_uri = "mysql://root:1234@:/test";
+//    options.path = "/tmp/vecwise_test";
     try {
         options = getDBMetaOptions();
     } catch(std::exception& ex) {
@@ -32,38 +45,203 @@ TEST_F(MySQLTest, TABLE_TEST) {
 
     int mode = Options::MODE::SINGLE;
     meta::MySQLMetaImpl impl(options, mode);
+//    auto status = impl.Initialize();
+//    ASSERT_TRUE(status.ok());
 
-    auto table_id = "meta_test_table";
+    meta::TableSchema schema1;
+    schema1.table_id_ = "test1";
+    schema1.dimension_ = 123;
 
-    meta::TableSchema table;
-    table.table_id_ = table_id;
-    auto status = impl.CreateTable(table);
+    auto status = impl.CreateTable(schema1);
+//    std::cout << status.ToString() << std::endl;
     ASSERT_TRUE(status.ok());
 
-    auto gid = table.id_;
-    table.id_ = -1;
-    status = impl.DescribeTable(table);
+    meta::TableSchema schema2;
+    schema2.table_id_ = "test2";
+    schema2.dimension_ = 321;
+    status = impl.CreateTable(schema2);
+//    std::cout << status.ToString() << std::endl;
     ASSERT_TRUE(status.ok());
-    ASSERT_EQ(table.id_, gid);
-    ASSERT_EQ(table.table_id_, table_id);
 
-    table.table_id_ = "not_found";
-    status = impl.DescribeTable(table);
+    status = impl.CreateTable(schema2);
+//    std::cout << status.ToString() << std::endl;
+//    ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery);
+    ASSERT_TRUE(status.ok());
+
+    status = impl.DeleteTable(schema2.table_id_);
+//    std::cout << status.ToString() << std::endl;
+    ASSERT_TRUE(status.ok());
+
+    size_t id1 = schema1.id_;
+    long created_on1 = schema1.created_on_;
+    status = impl.DescribeTable(schema1);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(schema1.id_, id1);
+    ASSERT_EQ(schema1.table_id_, "test1");
+    ASSERT_EQ(schema1.created_on_, created_on1);
+    ASSERT_EQ(schema1.files_cnt_, 0);
+    ASSERT_EQ(schema1.engine_type_, 1);
+    ASSERT_EQ(schema1.store_raw_data_, false);
+
+    bool check;
+    status = impl.HasTable("test1", check);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(check, true);
+
+    std::vector<meta::TableSchema> table_schema_array;
+    status = impl.AllTables(table_schema_array);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(table_schema_array.size(), 1);
+    meta::TableSchema resultSchema = table_schema_array[0];
+    ASSERT_EQ(resultSchema.id_, id1);
+    ASSERT_EQ(resultSchema.table_id_, "test1");
+    ASSERT_EQ(resultSchema.dimension_, 123);
+    ASSERT_EQ(resultSchema.files_cnt_, 0);
+    ASSERT_EQ(resultSchema.engine_type_, 1);
+    ASSERT_EQ(resultSchema.store_raw_data_, false);
+
+    meta::TableFileSchema tableFileSchema;
+    tableFileSchema.table_id_ = "test1";
+
+    status = impl.CreateTableFile(tableFileSchema);
+//    std::cout << status.ToString() << std::endl;
+    ASSERT_TRUE(status.ok());
+
+    tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX;
+    status = impl.UpdateTableFile(tableFileSchema);
+//    std::cout << status.ToString() << std::endl;
+    ASSERT_TRUE(status.ok());
+
+    meta::TableFilesSchema filesToIndex;
+    status = impl.FilesToIndex(filesToIndex);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(filesToIndex.size(), 1);
+    meta::TableFileSchema fileToIndex = filesToIndex[0];
+    ASSERT_EQ(fileToIndex.table_id_, "test1");
+    ASSERT_EQ(fileToIndex.dimension_, 123);
+
+//    meta::TableFilesSchema filesToIndex;
+//    status = impl.FilesToIndex(filesToIndex);
+//    ASSERT_TRUE(status.ok());
+//    ASSERT_EQ(filesToIndex.size(), 0);
+
+    meta::DatesT partition;
+    partition.push_back(tableFileSchema.date_);
+    meta::DatePartionedTableFilesSchema filesToSearch;
+    status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(filesToSearch.size(), 1);
+    ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1);
+    meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0];
+    ASSERT_EQ(fileToSearch.table_id_, "test1");
+    ASSERT_EQ(fileToSearch.dimension_, 123);
+
+    tableFileSchema.file_type_ = meta::TableFileSchema::RAW;
+    status = impl.UpdateTableFile(tableFileSchema);
+    ASSERT_TRUE(status.ok());
+
+    meta::DatePartionedTableFilesSchema filesToMerge;
+    status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge);
+//    std::cout << status.ToString() << std::endl;
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(filesToMerge.size(), 1);
+    ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1);
+    meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0];
+    ASSERT_EQ(fileToMerge.table_id_, "test1");
+    ASSERT_EQ(fileToMerge.dimension_, 123);
+
+    meta::TableFilesSchema resultTableFilesSchema;
+    std::vector<size_t> ids;
+    ids.push_back(tableFileSchema.id_);
+    status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(resultTableFilesSchema.size(), 1);
+    meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0];
+//    ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_);
+    ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_);
+    ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_);
+    ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_);
+    ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_);
+    ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_);
+    ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_);
+    ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_);
+
+    tableFileSchema.size_ = 234;
+    meta::TableSchema schema3;
+    schema3.table_id_ = "test3";
+    schema3.dimension_ = 321;
+    status = impl.CreateTable(schema3);
+    ASSERT_TRUE(status.ok());
+    meta::TableFileSchema tableFileSchema2;
+    tableFileSchema2.table_id_ = "test3";
+    tableFileSchema2.size_ = 345;
+    status = impl.CreateTableFile(tableFileSchema2);
+    ASSERT_TRUE(status.ok());
+    meta::TableFilesSchema filesToUpdate;
+    filesToUpdate.emplace_back(tableFileSchema);
+    filesToUpdate.emplace_back(tableFileSchema2);
+    status = impl.UpdateTableFile(tableFileSchema);
+    ASSERT_TRUE(status.ok());
+
+    uint64_t resultSize;
+    status = impl.Size(resultSize);
+//    std::cout << status.ToString() << std::endl;
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_);
+
+    uint64_t countResult;
+    status = impl.Count(tableFileSchema.table_id_, countResult);
+    ASSERT_TRUE(status.ok());
+
+    status = impl.DropAll();
+    ASSERT_TRUE(status.ok());
+
+}
+
+TEST_F(MySQLTest, GROUP_TEST) {
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
+
+    int mode = Options::MODE::SINGLE;
+    meta::MySQLMetaImpl impl(options, mode);
+
+    auto table_id = "meta_test_group";
+
+    meta::TableSchema group;
+    group.table_id_ = table_id;
+    auto status = impl.CreateTable(group);
+    ASSERT_TRUE(status.ok());
+
+    auto gid = group.id_;
+    group.id_ = -1;
+    status = impl.DescribeTable(group);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(group.id_, gid);
+    ASSERT_EQ(group.table_id_, table_id);
+
+    group.table_id_ = "not_found";
+    status = impl.DescribeTable(group);
     ASSERT_TRUE(!status.ok());
 
-    table.table_id_ = table_id;
-    status = impl.CreateTable(table);
+    group.table_id_ = table_id;
+    status = impl.CreateTable(group);
     ASSERT_TRUE(status.ok());
 
-    table.table_id_ = "";
-    status = impl.CreateTable(table);
+    group.table_id_ = "";
+    status = impl.CreateTable(group);
     ASSERT_TRUE(status.ok());
 
+
     status = impl.DropAll();
     ASSERT_TRUE(status.ok());
 }
 
-TEST_F(MySQLTest, TABLE_FILE_TEST) {
+TEST_F(MySQLTest, table_file_TEST) {
     DBMetaOptions options;
     try {
         options = getDBMetaOptions();
@@ -75,16 +253,17 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) {
     int mode = Options::MODE::SINGLE;
     meta::MySQLMetaImpl impl(options, mode);
 
-    auto table_id = "meta_test_table";
+    auto table_id = "meta_test_group";
 
-    meta::TableSchema table;
-    table.table_id_ = table_id;
-    table.dimension_ = 256;
-    auto status = impl.CreateTable(table);
+    meta::TableSchema group;
+    group.table_id_ = table_id;
+    group.dimension_ = 256;
+    auto status = impl.CreateTable(group);
 
     meta::TableFileSchema table_file;
-    table_file.table_id_ = table.table_id_;
+    table_file.table_id_ = group.table_id_;
     status = impl.CreateTableFile(table_file);
+//    std::cout << status.ToString() << std::endl;
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
 
@@ -153,15 +332,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
     int mode = Options::MODE::SINGLE;
     meta::MySQLMetaImpl impl(options, mode);
 
-    auto table_id = "meta_test_table";
+    auto table_id = "meta_test_group";
 
-    meta::TableSchema table;
-    table.table_id_ = table_id;
-    auto status = impl.CreateTable(table);
+    meta::TableSchema group;
+    group.table_id_ = table_id;
+    auto status = impl.CreateTable(group);
 
     meta::TableFilesSchema files;
     meta::TableFileSchema table_file;
-    table_file.table_id_ = table.table_id_;
+    table_file.table_id_ = group.table_id_;
 
     auto cnt = 100;
     long ts = utils::GetMicroSecTimeStamp();
@@ -212,13 +391,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
     auto impl = meta::MySQLMetaImpl(options, mode);
     auto table_id = "meta_test_group";
 
-    meta::TableSchema table;
-    table.table_id_ = table_id;
-    auto status = impl.CreateTable(table);
+    meta::TableSchema group;
+    group.table_id_ = table_id;
+    auto status = impl.CreateTable(group);
 
     meta::TableFilesSchema files;
     meta::TableFileSchema table_file;
-    table_file.table_id_ = table.table_id_;
+    table_file.table_id_ = group.table_id_;
 
     auto cnt = 10;
     auto each_size = 2UL;
@@ -266,9 +445,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
 
     auto table_id = "meta_test_group";
 
-    meta::TableSchema table;
-    table.table_id_ = table_id;
-    auto status = impl.CreateTable(table);
+    meta::TableSchema group;
+    group.table_id_ = table_id;
+    auto status = impl.CreateTable(group);
 
     int new_files_cnt = 4;
     int raw_files_cnt = 5;
@@ -276,7 +455,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
     int index_files_cnt = 7;
 
     meta::TableFileSchema table_file;
-    table_file.table_id_ = table.table_id_;
+    table_file.table_id_ = group.table_id_;
 
     for (auto i=0; i<new_files_cnt; ++i) {
         status = impl.CreateTableFile(table_file);
@@ -309,7 +488,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
     ASSERT_EQ(files.size(), to_index_files_cnt);
 
     meta::DatePartionedTableFilesSchema dated_files;
-    status = impl.FilesToMerge(table.table_id_, dated_files);
+    status = impl.FilesToMerge(group.table_id_, dated_files);
     ASSERT_TRUE(status.ok());
     ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);