Merge branch 'branch-0.5.0' into 'branch-0.5.0'

refine code

See merge request megasearch/milvus!664

Former-commit-id: ab4a168e0464af06b0dd9f989bc4ee207f237e38

cpp/src/db/meta/MetaConsts.h

@@ -21,11 +21,6 @@ namespace milvus {
 namespace engine {
 namespace meta {
 
-const size_t K = 1024UL;
-const size_t M = K * K;
-const size_t G = K * M;
-const size_t T = K * G;
-
 const size_t S_PS = 1UL;
 const size_t MS_PS = 1000 * S_PS;
 const size_t US_PS = 1000 * MS_PS;

cpp/src/scheduler/ResourceMgr.cpp

@@ -79,9 +79,7 @@ ResourceMgr::Add(ResourcePtr&& resource) {
             gpu_resources_.emplace_back(ResourceWPtr(resource));
             break;
         }
-        default: {
-            break;
-        }
+        default: { break; }
     }
     resources_.emplace_back(resource);
 
@@ -200,13 +198,19 @@ bool
 ResourceMgr::check_resource_valid() {
     {
         // TODO: check one disk-resource, one cpu-resource, zero or more gpu-resource;
-        if (GetDiskResources().size() != 1) return false;
-        if (GetCpuResources().size() != 1) return false;
+        if (GetDiskResources().size() != 1) {
+            return false;
+        }
+        if (GetCpuResources().size() != 1) {
+            return false;
+        }
     }
 
     {
         // TODO: one compute-resource at least;
-        if (GetNumOfComputeResource() < 1) return false;
+        if (GetNumOfComputeResource() < 1) {
+            return false;
+        }
     }
 
     {
@@ -237,9 +241,7 @@ void
 ResourceMgr::event_process() {
     while (running_) {
         std::unique_lock<std::mutex> lock(event_mutex_);
-        event_cv_.wait(lock, [this] {
-            return !queue_.empty();
-        });
+        event_cv_.wait(lock, [this] { return !queue_.empty(); });
 
         auto event = queue_.front();
         queue_.pop();

cpp/src/scheduler/task/SearchTask.cpp

@@ -22,6 +22,7 @@
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 
+#include <algorithm>
 #include <string>
 #include <thread>
 #include <utility>
@@ -230,13 +231,8 @@ XSearchTask::Execute() {
 }
 
 Status
-XSearchTask::TopkResult(const std::vector<long> &input_ids,
-                        const std::vector<float> &input_distance,
-                        uint64_t input_k,
-                        uint64_t nq,
-                        uint64_t topk,
-                        bool ascending,
-                        scheduler::ResultSet &result) {
+XSearchTask::TopkResult(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance,
+                        uint64_t input_k, uint64_t nq, uint64_t topk, bool ascending, scheduler::ResultSet& result) {
     scheduler::ResultSet result_buf;
 
     if (result.empty()) {
@@ -266,7 +262,7 @@ XSearchTask::TopkResult(const std::vector<long> &input_ids,
                 auto& result_buf_item = result_buf_i[buf_k];
                 auto& result_item = result_i[tar_k];
                 if ((ascending && input_distance[src_idx] < result_item.second) ||
-                   (!ascending && input_distance[src_idx] > result_item.second)) {
+                    (!ascending && input_distance[src_idx] > result_item.second)) {
                     result_buf_item.first = input_ids[src_idx];
                     result_buf_item.second = input_distance[src_idx];
                     src_k++;

cpp/src/scheduler/task/SearchTask.h

@@ -39,13 +39,8 @@ class XSearchTask : public Task {
 
  public:
     static Status
-    TopkResult(const std::vector<long> &input_ids,
-               const std::vector<float> &input_distance,
-               uint64_t input_k,
-               uint64_t nq,
-               uint64_t topk,
-               bool ascending,
-               scheduler::ResultSet &result);
+    TopkResult(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance, uint64_t input_k,
+               uint64_t nq, uint64_t topk, bool ascending, scheduler::ResultSet& result);
 
  public:
     TableFileSchemaPtr file_;

cpp/unittest/db/test_db.cpp

@@ -18,6 +18,7 @@
 #include "db/utils.h"
 #include "db/DB.h"
 #include "db/DBImpl.h"
+#include "db/Constants.h"
 #include "db/meta/MetaConsts.h"
 #include "db/DBFactory.h"
 #include "cache/CpuCacheMgr.h"
@@ -190,7 +191,7 @@ TEST_F(DBTest, DB_TEST) {
 
             START_TIMER;
             stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
-            ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
+            ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
             STOP_TIMER(ss.str());
 
             ASSERT_TRUE(stat.ok());
@@ -439,7 +440,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
 
     db_->Size(size);
     LOG(DEBUG) << "size=" << size;
-    ASSERT_LE(size, 1 * ms::engine::meta::G);
+    ASSERT_LE(size, 1 * ms::engine::G);
 }
 
 TEST_F(DBTest2, DELETE_TEST) {

cpp/unittest/db/test_db_mysql.cpp

@@ -18,6 +18,7 @@
 #include "db/utils.h"
 #include "db/DB.h"
 #include "db/DBImpl.h"
+#include "db/Constants.h"
 #include "db/meta/MetaConsts.h"
 
 #include <gtest/gtest.h>
@@ -98,7 +99,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
 
             START_TIMER;
             stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
-            ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
+            ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
             STOP_TIMER(ss.str());
 
             ASSERT_TRUE(stat.ok());
@@ -236,7 +237,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
 
     db_->Size(size);
     LOG(DEBUG) << "size=" << size;
-    ASSERT_LE(size, 1 * ms::engine::meta::G);
+    ASSERT_LE(size, 1 * ms::engine::G);
 }
 
 TEST_F(MySqlDBTest, DELETE_TEST) {

cpp/unittest/db/test_mem.cpp

@@ -329,7 +329,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
 
             START_TIMER;
             stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results);
-            ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
+            ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
             STOP_TIMER(ss.str());
 
             ASSERT_TRUE(stat.ok());

cpp/unittest/db/test_meta.cpp

@@ -18,6 +18,7 @@
 #include "db/utils.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/Utils.h"
+#include "db/Constants.h"
 #include "db/meta/MetaConsts.h"
 
 #include <gtest/gtest.h>
@@ -192,7 +193,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
     for (auto i = 0; i < cnt; ++i) {
         status = impl.CreateTableFile(table_file);
         table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
-        table_file.file_size_ = each_size * ms::engine::meta::G;
+        table_file.file_size_ = each_size * ms::engine::G;
         status = impl.UpdateTableFile(table_file);
         files.push_back(table_file);
         ids.push_back(table_file.id_);

cpp/unittest/db/test_meta_mysql.cpp

@@ -213,7 +213,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
     for (auto i = 0; i < cnt; ++i) {
         status = impl.CreateTableFile(table_file);
         table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
-        table_file.file_size_ = each_size * ms::engine::meta::G;
+        table_file.file_size_ = each_size * ms::engine::G;
         status = impl.UpdateTableFile(table_file);
         files.push_back(table_file);
         ids.push_back(table_file.id_);

cpp/unittest/db/test_search.cpp

@@ -22,10 +22,10 @@
 #include "scheduler/task/SearchTask.h"
 #include "utils/TimeRecorder.h"
 
-using namespace milvus::scheduler;
-
 namespace {
 
+namespace ms = milvus::scheduler;
+
 void
 BuildResult(uint64_t nq,
             uint64_t topk,
@@ -45,14 +45,14 @@ BuildResult(uint64_t nq,
     }
 }
 
-void CheckTopkResult(const std::vector<long> &input_ids_1,
+void CheckTopkResult(const std::vector<int64_t> &input_ids_1,
                      const std::vector<float> &input_distance_1,
-                     const std::vector<long> &input_ids_2,
+                     const std::vector<int64_t> &input_ids_2,
                      const std::vector<float> &input_distance_2,
                      uint64_t nq,
                      uint64_t topk,
                      bool ascending,
-                     const ResultSet& result) {
+                     const ms::ResultSet& result) {
     ASSERT_EQ(result.size(), nq);
     ASSERT_EQ(input_ids_1.size(), input_distance_1.size());
     ASSERT_EQ(input_ids_2.size(), input_distance_2.size());
@@ -85,21 +85,21 @@ TEST(DBSearchTest, TOPK_TEST) {
     uint64_t NQ = 15;
     uint64_t TOP_K = 64;
     bool ascending;
-    std::vector<long> ids1, ids2;
+    std::vector<int64_t> ids1, ids2;
     std::vector<float> dist1, dist2;
-    ResultSet result;
+    ms::ResultSet result;
     milvus::Status status;
 
     /* test1, id1/dist1 valid, id2/dist2 empty */
     ascending = true;
     BuildResult(NQ, TOP_K, ascending, ids1, dist1);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
     /* test2, id1/dist1 valid, id2/dist2 valid */
     BuildResult(NQ, TOP_K, ascending, ids2, dist2);
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
@@ -108,9 +108,9 @@ TEST(DBSearchTest, TOPK_TEST) {
     dist1.clear();
     result.clear();
     BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
@@ -119,9 +119,9 @@ TEST(DBSearchTest, TOPK_TEST) {
     dist2.clear();
     result.clear();
     BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
@@ -135,13 +135,13 @@ TEST(DBSearchTest, TOPK_TEST) {
 
     /* test1, id1/dist1 valid, id2/dist2 empty */
     BuildResult(NQ, TOP_K, ascending, ids1, dist1);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
     /* test2, id1/dist1 valid, id2/dist2 valid */
     BuildResult(NQ, TOP_K, ascending, ids2, dist2);
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
@@ -150,9 +150,9 @@ TEST(DBSearchTest, TOPK_TEST) {
     dist1.clear();
     result.clear();
     BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 
@@ -161,9 +161,9 @@ TEST(DBSearchTest, TOPK_TEST) {
     dist2.clear();
     result.clear();
     BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
-    status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
-    status = XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
+    status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
     ASSERT_TRUE(status.ok());
     CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
 }
@@ -173,9 +173,9 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
     int32_t top_k = 1000;
     int32_t index_file_num = 478;   /* sift1B dataset, index files num */
     bool ascending = true;
-    std::vector<long> input_ids;
+    std::vector<int64_t> input_ids;
     std::vector<float> input_distance;
-    ResultSet final_result;
+    ms::ResultSet final_result;
     milvus::Status status;
 
     double span, reduce_cost = 0.0;
@@ -187,7 +187,7 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
         rc.RecordSection("do search for context: " + std::to_string(i));
 
         // pick up topk result
-        status = XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result);
+        status = ms::XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result);
         ASSERT_TRUE(status.ok());
         ASSERT_EQ(final_result.size(), nq);