remove MetaTypes.h (#3072)

* remove MetaTypes.h
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* refine code
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* remove MetaConsts.h
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

core/src/codecs/StructuredIndexFormat.cpp

@@ -24,7 +24,7 @@
 #include <memory>
 #include <utility>
 
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 #include "knowhere/index/structured_index/StructuredIndexSort.h"
 
 #include "utils/Exception.h"
@@ -43,30 +43,30 @@ StructuredIndexFormat::FilePostfix() {
 }
 
 knowhere::IndexPtr
-StructuredIndexFormat::CreateStructuredIndex(const milvus::engine::meta::DataType data_type) {
+StructuredIndexFormat::CreateStructuredIndex(const engine::DataType data_type) {
     knowhere::IndexPtr index = nullptr;
     switch (data_type) {
-        case engine::meta::DataType::INT8: {
+        case engine::DataType::INT8: {
             index = std::make_shared<knowhere::StructuredIndexSort<int8_t>>();
             break;
         }
-        case engine::meta::DataType::INT16: {
+        case engine::DataType::INT16: {
             index = std::make_shared<knowhere::StructuredIndexSort<int16_t>>();
             break;
         }
-        case engine::meta::DataType::INT32: {
+        case engine::DataType::INT32: {
             index = std::make_shared<knowhere::StructuredIndexSort<int32_t>>();
             break;
         }
-        case engine::meta::DataType::INT64: {
+        case engine::DataType::INT64: {
             index = std::make_shared<knowhere::StructuredIndexSort<int64_t>>();
             break;
         }
-        case engine::meta::DataType::FLOAT: {
+        case engine::DataType::FLOAT: {
             index = std::make_shared<knowhere::StructuredIndexSort<float>>();
             break;
         }
-        case engine::meta::DataType::DOUBLE: {
+        case engine::DataType::DOUBLE: {
             index = std::make_shared<knowhere::StructuredIndexSort<double>>();
             break;
         }
@@ -135,14 +135,14 @@ StructuredIndexFormat::Read(const milvus::storage::FSHandlerPtr& fs_ptr, const s
     double rate = length * 1000000.0 / span / 1024 / 1024;
     LOG_ENGINE_DEBUG_ << "StructuredIndexFormat::read(" << full_file_path << ") rate " << rate << "MB/s";
 
-    auto attr_type = static_cast<engine::meta::DataType>(data_type);
+    auto attr_type = static_cast<engine::DataType>(data_type);
     index = CreateStructuredIndex(attr_type);
     index->Load(load_data_list);
 }
 
 void
 StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const std::string& file_path,
-                             engine::meta::DataType data_type, const knowhere::IndexPtr& index) {
+                             engine::DataType data_type, const knowhere::IndexPtr& index) {
     milvus::TimeRecorder recorder("StructuredIndexFormat::Write");
 
     std::string full_file_path = file_path + STRUCTURED_INDEX_POSTFIX;

core/src/codecs/StructuredIndexFormat.h

@@ -21,7 +21,7 @@
 #include <string>
 #include <vector>
 
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 #include "knowhere/index/Index.h"
 #include "storage/FSHandler.h"
 
@@ -39,7 +39,7 @@ class StructuredIndexFormat {
     Read(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path, knowhere::IndexPtr& index);
 
     void
-    Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path, engine::meta::DataType data_type,
+    Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path, engine::DataType data_type,
           const knowhere::IndexPtr& index);
 
     // No copy and move
@@ -53,7 +53,7 @@ class StructuredIndexFormat {
 
  private:
     knowhere::IndexPtr
-    CreateStructuredIndex(const engine::meta::DataType data_type);
+    CreateStructuredIndex(const engine::DataType data_type);
 };
 
 using StructuredIndexFormatPtr = std::shared_ptr<StructuredIndexFormat>;

core/src/db/DBImpl.cpp

@@ -212,7 +212,7 @@ DBImpl::CreateCollection(const snapshot::CreateCollectionContext& context) {
     // check uid existence/validation
     bool has_uid = false;
     for (auto& pair : ctx.fields_schema) {
-        if (pair.first->GetFtype() == meta::DataType::UID) {
+        if (pair.first->GetFtype() == DataType::UID) {
             has_uid = true;
             break;
         }
@@ -220,7 +220,7 @@ DBImpl::CreateCollection(const snapshot::CreateCollectionContext& context) {
 
     // add uid field if not specified
     if (!has_uid) {
-        auto uid_field = std::make_shared<snapshot::Field>(DEFAULT_UID_NAME, 0, milvus::engine::FieldType::UID);
+        auto uid_field = std::make_shared<snapshot::Field>(DEFAULT_UID_NAME, 0, milvus::engine::DataType::UID);
         auto bloom_filter_element = std::make_shared<snapshot::FieldElement>(
             0, 0, DEFAULT_BLOOM_FILTER_NAME, milvus::engine::FieldElementType::FET_BLOOM_FILTER);
         auto delete_doc_element = std::make_shared<snapshot::FieldElement>(

core/src/db/SnapshotHandlers.cpp

@@ -12,8 +12,6 @@
 #include "db/SnapshotHandlers.h"
 #include "db/SnapshotVisitor.h"
 #include "db/Types.h"
-#include "db/meta/MetaConsts.h"
-#include "db/meta/MetaTypes.h"
 #include "db/snapshot/ResourceHelper.h"
 #include "db/snapshot/Resources.h"
 #include "db/snapshot/Snapshot.h"
@@ -34,8 +32,7 @@ LoadVectorFieldElementHandler::LoadVectorFieldElementHandler(const std::shared_p
 
 Status
 LoadVectorFieldElementHandler::Handle(const snapshot::FieldElementPtr& field_element) {
-    if (field_->GetFtype() != engine::FieldType::VECTOR_FLOAT &&
-        field_->GetFtype() != engine::FieldType::VECTOR_BINARY) {
+    if (field_->GetFtype() != engine::DataType::VECTOR_FLOAT && field_->GetFtype() != engine::DataType::VECTOR_BINARY) {
         return Status(DB_ERROR, "Should be VECTOR field");
     }
     if (field_->GetID() != field_element->GetFieldId()) {
@@ -52,7 +49,7 @@ LoadVectorFieldHandler::LoadVectorFieldHandler(const std::shared_ptr<server::Con
 
 Status
 LoadVectorFieldHandler::Handle(const snapshot::FieldPtr& field) {
-    if (field->GetFtype() != engine::FieldType::VECTOR_FLOAT && field->GetFtype() != engine::FieldType::VECTOR_BINARY) {
+    if (field->GetFtype() != engine::DataType::VECTOR_FLOAT && field->GetFtype() != engine::DataType::VECTOR_BINARY) {
         return Status::OK();
     }
     if (context_ && context_->IsConnectionBroken()) {
@@ -93,7 +90,7 @@ SegmentsToIndexCollector::SegmentsToIndexCollector(snapshot::ScopedSnapshotT ss,
 
 Status
 SegmentsToIndexCollector::Handle(const snapshot::SegmentCommitPtr& segment_commit) {
-    if (segment_commit->GetRowCount() < meta::BUILD_INDEX_THRESHOLD) {
+    if (segment_commit->GetRowCount() < engine::BUILD_INDEX_THRESHOLD) {
         return Status::OK();
     }
 

core/src/db/SnapshotVisitor.cpp

@@ -11,8 +11,9 @@
 
 #include "db/SnapshotVisitor.h"
 #include <sstream>
+
 #include "db/SnapshotHandlers.h"
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 #include "db/snapshot/Snapshots.h"
 
 namespace milvus {

core/src/db/Types.h

@@ -22,21 +22,42 @@
 #include <utility>
 #include <vector>
 
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 #include "utils/Json.h"
 
 namespace milvus {
 namespace engine {
 
-typedef int64_t IDNumber;
-typedef IDNumber* IDNumberPtr;
-typedef std::vector<IDNumber> IDNumbers;
+using DateT = int;
 
-typedef faiss::Index::distance_t VectorDistance;
-typedef std::vector<VectorDistance> VectorDistances;
+using IDNumber = int64_t;
+using IDNumberPtr = IDNumber*;
+using IDNumbers = std::vector<IDNumber>;
 
-typedef std::vector<faiss::Index::idx_t> ResultIds;
-typedef std::vector<faiss::Index::distance_t> ResultDistances;
+using VectorDistance = faiss::Index::distance_t;
+using VectorDistances = std::vector<VectorDistance>;
+
+using ResultIds = std::vector<faiss::Index::idx_t>;
+using ResultDistances = std::vector<faiss::Index::distance_t>;
+
+enum DataType {
+    NONE = 0,
+    BOOL = 1,
+    INT8 = 2,
+    INT16 = 3,
+    INT32 = 4,
+    INT64 = 5,
+
+    FLOAT = 10,
+    DOUBLE = 11,
+
+    STRING = 20,
+
+    UID = 30,
+
+    VECTOR_BINARY = 100,
+    VECTOR_FLOAT = 101,
+};
 
 struct CollectionIndex {
     std::string index_name_;
@@ -60,7 +81,7 @@ struct Entity {
 
 struct AttrsData {
     uint64_t attr_count_ = 0;
-    std::unordered_map<std::string, engine::meta::DataType> attr_type_;
+    std::unordered_map<std::string, engine::DataType> attr_type_;
     std::unordered_map<std::string, std::vector<uint8_t>> attr_data_;
     IDNumbers id_array_;
 };
@@ -90,15 +111,14 @@ extern const char* PARAM_INDEX_METRIC_TYPE;
 extern const char* PARAM_INDEX_EXTRA_PARAMS;
 extern const char* PARAM_SEGMENT_ROW_COUNT;
 
+constexpr int64_t BUILD_INDEX_THRESHOLD = 4096;  // row count threshold when building index
 constexpr int64_t MAX_NAME_LENGTH = 255;
 constexpr int64_t MAX_DIMENSION = 32768;
 constexpr int32_t MAX_SEGMENT_ROW_COUNT = 4 * 1024 * 1024;
-constexpr int64_t DEFAULT_SEGMENT_ROW_COUNT = 100000;
+constexpr int64_t DEFAULT_SEGMENT_ROW_COUNT = 100000;  // default row count per segment when creating collection
 constexpr int64_t M_BYTE = 1024 * 1024;
 constexpr int64_t MAX_INSERT_DATA_SIZE = 256 * M_BYTE;
 
-using FieldType = meta::DataType;
-
 enum FieldElementType {
     FET_NONE = 0,
     FET_RAW = 1,

core/src/db/Utils.cpp

@@ -60,7 +60,7 @@ IsBinaryMetricType(const std::string& metric_type) {
            (metric_type == knowhere::Metric::TANIMOTO);
 }
 
-meta::DateT
+engine::DateT
 GetDate(const std::time_t& t, int day_delta) {
     struct tm ltm;
     localtime_r(&t, &ltm);
@@ -80,12 +80,12 @@ GetDate(const std::time_t& t, int day_delta) {
     return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday;
 }
 
-meta::DateT
+engine::DateT
 GetDateWithDelta(int day_delta) {
     return GetDate(std::time(nullptr), day_delta);
 }
 
-meta::DateT
+engine::DateT
 GetDate() {
     return GetDate(std::time(nullptr), 0);
 }

core/src/db/Utils.h

@@ -16,7 +16,6 @@
 
 #include "Options.h"
 #include "db/Types.h"
-#include "db/meta/MetaTypes.h"
 #include "utils/Status.h"
 
 namespace milvus {
@@ -37,11 +36,11 @@ IsSameIndex(const CollectionIndex& index1, const CollectionIndex& index2);
 bool
 IsBinaryMetricType(const std::string& metric_type);
 
-meta::DateT
+engine::DateT
 GetDate(const std::time_t& t, int day_delta = 0);
-meta::DateT
+engine::DateT
 GetDate();
-meta::DateT
+engine::DateT
 GetDateWithDelta(int day_delta);
 
 struct MetaUriInfo {

core/src/db/engine/ExecutionEngine.h

@@ -19,7 +19,6 @@
 #include <vector>
 
 #include "db/Types.h"
-#include "db/meta/MetaTypes.h"
 #include "query/GeneralQuery.h"
 #include "utils/Status.h"
 

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -89,10 +89,10 @@ ExecutionEngineImpl::LoadForSearch(const query::QueryPtr& query_ptr) {
 }
 
 Status
-ExecutionEngineImpl::CreateStructuredIndex(const milvus::engine::meta::DataType field_type,
-                                           std::vector<uint8_t>& raw_data, knowhere::IndexPtr& index_ptr) {
+ExecutionEngineImpl::CreateStructuredIndex(const DataType field_type, std::vector<uint8_t>& raw_data,
+                                           knowhere::IndexPtr& index_ptr) {
     switch (field_type) {
-        case engine::meta::DataType::INT32: {
+        case engine::DataType::INT32: {
             auto size = raw_data.size() / sizeof(int32_t);
             std::vector<int32_t> int32_data(size, 0);
             memcpy(int32_data.data(), raw_data.data(), size);
@@ -101,20 +101,20 @@ ExecutionEngineImpl::CreateStructuredIndex(const milvus::engine::meta::DataType
             index_ptr = std::static_pointer_cast<knowhere::Index>(int32_index_ptr);
             break;
         }
-        case engine::meta::DataType::UID:
-        case engine::meta::DataType::INT64: {
+        case engine::DataType::UID:
+        case engine::DataType::INT64: {
             auto int64_index_ptr = std::make_shared<knowhere::StructuredIndexSort<int64_t>>(
                 raw_data.size(), reinterpret_cast<const int64_t*>(raw_data.data()));
             index_ptr = std::static_pointer_cast<knowhere::Index>(int64_index_ptr);
             break;
         }
-        case engine::meta::DataType::FLOAT: {
+        case engine::DataType::FLOAT: {
             auto float_index_ptr = std::make_shared<knowhere::StructuredIndexSort<float>>(
                 raw_data.size(), reinterpret_cast<const float*>(raw_data.data()));
             index_ptr = std::static_pointer_cast<knowhere::Index>(float_index_ptr);
             break;
         }
-        case engine::meta::DataType::DOUBLE: {
+        case engine::DataType::DOUBLE: {
             auto double_index_ptr = std::make_shared<knowhere::StructuredIndexSort<double>>(
                 raw_data.size(), reinterpret_cast<const double*>(raw_data.data()));
             index_ptr = std::static_pointer_cast<knowhere::Index>(double_index_ptr);
@@ -279,7 +279,7 @@ ExecutionEngineImpl::Search(ExecutionEngineContext& context) {
         SegmentPtr segment_ptr;
         segment_reader_->GetSegment(segment_ptr);
         knowhere::VecIndexPtr vec_index = nullptr;
-        std::unordered_map<std::string, engine::meta::DataType> attr_type;
+        std::unordered_map<std::string, engine::DataType> attr_type;
 
         auto segment_visitor = segment_reader_->GetSegmentVisitor();
         auto field_visitors = segment_visitor->GetFieldVisitors();
@@ -287,13 +287,13 @@ ExecutionEngineImpl::Search(ExecutionEngineContext& context) {
             auto& field_visitor = pair.second;
             auto& field = field_visitor->GetField();
             auto type = field->GetFtype();
-            if (field->GetFtype() == (int)engine::meta::DataType::VECTOR_FLOAT ||
-                field->GetFtype() == (int)engine::meta::DataType::VECTOR_BINARY) {
+            if (field->GetFtype() == (int)engine::DataType::VECTOR_FLOAT ||
+                field->GetFtype() == (int)engine::DataType::VECTOR_BINARY) {
                 segment_ptr->GetVectorIndex(field->GetName(), vec_index);
-            } else if (type == (int)engine::meta::DataType::UID) {
+            } else if (type == (int)engine::DataType::UID) {
                 continue;
             } else {
-                attr_type.insert(std::make_pair(field->GetName(), (engine::meta::DataType)type));
+                attr_type.insert(std::make_pair(field->GetName(), (engine::DataType)type));
             }
         }
 
@@ -334,7 +334,7 @@ ExecutionEngineImpl::Search(ExecutionEngineContext& context) {
 Status
 ExecutionEngineImpl::ExecBinaryQuery(const milvus::query::GeneralQueryPtr& general_query,
                                      faiss::ConcurrentBitsetPtr& bitset,
-                                     std::unordered_map<std::string, meta::DataType>& attr_type,
+                                     std::unordered_map<std::string, DataType>& attr_type,
                                      std::string& vector_placeholder) {
     Status status = Status::OK();
     if (general_query->leaf == nullptr) {
@@ -432,33 +432,33 @@ ProcessIndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, knowhere::IndexPtr&
 
 Status
 ExecutionEngineImpl::IndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, const std::string& field_name,
-                                      const meta::DataType& data_type, milvus::json& term_values_json) {
+                                      const DataType& data_type, milvus::json& term_values_json) {
     SegmentPtr segment_ptr;
     segment_reader_->GetSegment(segment_ptr);
     knowhere::IndexPtr index_ptr = nullptr;
     auto attr_index = segment_ptr->GetStructuredIndex(field_name, index_ptr);
     switch (data_type) {
-        case meta::DataType::INT8: {
+        case DataType::INT8: {
             ProcessIndexedTermQuery<int8_t>(bitset, index_ptr, term_values_json);
             break;
         }
-        case meta::DataType::INT16: {
+        case DataType::INT16: {
             ProcessIndexedTermQuery<int16_t>(bitset, index_ptr, term_values_json);
             break;
         }
-        case meta::DataType::INT32: {
+        case DataType::INT32: {
             ProcessIndexedTermQuery<int32_t>(bitset, index_ptr, term_values_json);
             break;
         }
-        case meta::DataType::INT64: {
+        case DataType::INT64: {
             ProcessIndexedTermQuery<int64_t>(bitset, index_ptr, term_values_json);
             break;
         }
-        case meta::DataType::FLOAT: {
+        case DataType::FLOAT: {
             ProcessIndexedTermQuery<float>(bitset, index_ptr, term_values_json);
             break;
         }
-        case meta::DataType::DOUBLE: {
+        case DataType::DOUBLE: {
             ProcessIndexedTermQuery<double>(bitset, index_ptr, term_values_json);
             break;
         }
@@ -469,7 +469,7 @@ ExecutionEngineImpl::IndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, const
 
 Status
 ExecutionEngineImpl::ProcessTermQuery(faiss::ConcurrentBitsetPtr& bitset, const query::TermQueryPtr& term_query,
-                                      std::unordered_map<std::string, meta::DataType>& attr_type) {
+                                      std::unordered_map<std::string, DataType>& attr_type) {
     auto status = Status::OK();
     auto term_query_json = term_query->json_obj;
     auto term_it = term_query_json.begin();
@@ -510,31 +510,31 @@ ProcessIndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, knowhere::IndexPtr&
 }
 
 Status
-ExecutionEngineImpl::IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const meta::DataType& data_type,
+ExecutionEngineImpl::IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const DataType& data_type,
                                        knowhere::IndexPtr& index_ptr, milvus::json& range_values_json) {
     auto status = Status::OK();
     switch (data_type) {
-        case meta::DataType::INT8: {
+        case DataType::INT8: {
             ProcessIndexedRangeQuery<int8_t>(bitset, index_ptr, range_values_json);
             break;
         }
-        case meta::DataType::INT16: {
+        case DataType::INT16: {
             ProcessIndexedRangeQuery<int16_t>(bitset, index_ptr, range_values_json);
             break;
         }
-        case meta::DataType::INT32: {
+        case DataType::INT32: {
             ProcessIndexedRangeQuery<int32_t>(bitset, index_ptr, range_values_json);
             break;
         }
-        case meta::DataType::INT64: {
+        case DataType::INT64: {
             ProcessIndexedRangeQuery<int64_t>(bitset, index_ptr, range_values_json);
             break;
         }
-        case meta::DataType::FLOAT: {
+        case DataType::FLOAT: {
             ProcessIndexedRangeQuery<float>(bitset, index_ptr, range_values_json);
             break;
         }
-        case meta::DataType::DOUBLE: {
+        case DataType::DOUBLE: {
             ProcessIndexedRangeQuery<double>(bitset, index_ptr, range_values_json);
             break;
         }
@@ -545,7 +545,7 @@ ExecutionEngineImpl::IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const
 }
 
 Status
-ExecutionEngineImpl::ProcessRangeQuery(const std::unordered_map<std::string, meta::DataType>& attr_type,
+ExecutionEngineImpl::ProcessRangeQuery(const std::unordered_map<std::string, DataType>& attr_type,
                                        faiss::ConcurrentBitsetPtr& bitset, const query::RangeQueryPtr& range_query) {
     SegmentPtr segment_ptr;
     segment_reader_->GetSegment(segment_ptr);

core/src/db/engine/ExecutionEngineImpl.h

@@ -49,7 +49,7 @@ class ExecutionEngineImpl : public ExecutionEngine {
     CreateVecIndex(const std::string& index_name);
 
     Status
-    CreateStructuredIndex(const engine::meta::DataType field_type, std::vector<uint8_t>& raw_data,
+    CreateStructuredIndex(const engine::DataType field_type, std::vector<uint8_t>& raw_data,
                           knowhere::IndexPtr& index_ptr);
 
     Status
@@ -60,23 +60,23 @@ class ExecutionEngineImpl : public ExecutionEngine {
 
     Status
     ExecBinaryQuery(const query::GeneralQueryPtr& general_query, faiss::ConcurrentBitsetPtr& bitset,
-                    std::unordered_map<std::string, meta::DataType>& attr_type, std::string& vector_placeholder);
+                    std::unordered_map<std::string, DataType>& attr_type, std::string& vector_placeholder);
 
     Status
     ProcessTermQuery(faiss::ConcurrentBitsetPtr& bitset, const query::TermQueryPtr& term_query,
-                     std::unordered_map<std::string, meta::DataType>& attr_type);
+                     std::unordered_map<std::string, DataType>& attr_type);
 
     Status
-    IndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, const std::string& field_name, const meta::DataType& data_type,
+    IndexedTermQuery(faiss::ConcurrentBitsetPtr& bitset, const std::string& field_name, const DataType& data_type,
                      milvus::json& term_values_json);
 
     Status
-    ProcessRangeQuery(const std::unordered_map<std::string, meta::DataType>& attr_type,
-                      faiss::ConcurrentBitsetPtr& bitset, const query::RangeQueryPtr& range_query);
+    ProcessRangeQuery(const std::unordered_map<std::string, DataType>& attr_type, faiss::ConcurrentBitsetPtr& bitset,
+                      const query::RangeQueryPtr& range_query);
 
     Status
-    IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const meta::DataType& data_type,
-                      knowhere::IndexPtr& index_ptr, milvus::json& range_values_json);
+    IndexedRangeQuery(faiss::ConcurrentBitsetPtr& bitset, const DataType& data_type, knowhere::IndexPtr& index_ptr,
+                      milvus::json& range_values_json);
 
     using AddSegmentFileOperation = std::shared_ptr<snapshot::ChangeSegmentFileOperation>;
     Status

core/src/db/insert/MemManagerImpl.cpp

@@ -90,47 +90,47 @@ MemManagerImpl::ValidateChunk(int64_t collection_id, int64_t partition_id, const
         size_t data_size = iter->second.size();
 
         snapshot::FieldPtr field = ss->GetField(name);
-        meta::DataType ftype = static_cast<meta::DataType>(field->GetFtype());
+        DataType ftype = static_cast<DataType>(field->GetFtype());
         std::string err_msg = "Illegal data size for chunk field: ";
         switch (ftype) {
-            case meta::DataType::BOOL:
+            case DataType::BOOL:
                 if (data_size != chunk->count_ * sizeof(bool)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::DOUBLE:
+            case DataType::DOUBLE:
                 if (data_size != chunk->count_ * sizeof(double)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::FLOAT:
+            case DataType::FLOAT:
                 if (data_size != chunk->count_ * sizeof(float)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::INT8:
+            case DataType::INT8:
                 if (data_size != chunk->count_ * sizeof(uint8_t)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::INT16:
+            case DataType::INT16:
                 if (data_size != chunk->count_ * sizeof(uint16_t)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::INT32:
+            case DataType::INT32:
                 if (data_size != chunk->count_ * sizeof(uint32_t)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::UID:
-            case meta::DataType::INT64:
+            case DataType::UID:
+            case DataType::INT64:
                 if (data_size != chunk->count_ * sizeof(uint64_t)) {
                     return Status(DB_ERROR, err_msg + name);
                 }
                 break;
-            case meta::DataType::VECTOR_FLOAT:
-            case meta::DataType::VECTOR_BINARY: {
+            case DataType::VECTOR_FLOAT:
+            case DataType::VECTOR_BINARY: {
                 json params = field->GetParams();
                 if (params.find(knowhere::meta::DIM) == params.end()) {
                     std::string msg = "Vector field params must contain: dimension";
@@ -139,7 +139,7 @@ MemManagerImpl::ValidateChunk(int64_t collection_id, int64_t partition_id, const
                 }
 
                 int64_t dimension = params[knowhere::meta::DIM];
-                int64_t row_size = (ftype == meta::DataType::VECTOR_BINARY) ? dimension / 8 : dimension * sizeof(float);
+                int64_t row_size = (ftype == DataType::VECTOR_BINARY) ? dimension / 8 : dimension * sizeof(float);
                 if (data_size != chunk->count_ * row_size) {
                     return Status(DB_ERROR, err_msg + name);
                 }

core/src/db/insert/MemSegment.cpp

@@ -19,9 +19,8 @@
 
 #include "config/ServerConfig.h"
 #include "db/Constants.h"
+#include "db/Types.h"
 #include "db/Utils.h"
-#include "db/engine/EngineFactory.h"
-#include "db/meta/MetaTypes.h"
 #include "db/snapshot/Operations.h"
 #include "db/snapshot/Snapshots.h"
 #include "knowhere/index/vector_index/helpers/IndexParameter.h"
@@ -126,32 +125,32 @@ MemSegment::GetSingleEntitySize(int64_t& single_size) {
     std::vector<std::string> field_names = ss->GetFieldNames();
     for (auto& name : field_names) {
         snapshot::FieldPtr field = ss->GetField(name);
-        meta::DataType ftype = static_cast<meta::DataType>(field->GetFtype());
+        DataType ftype = static_cast<DataType>(field->GetFtype());
         switch (ftype) {
-            case meta::DataType::BOOL:
+            case DataType::BOOL:
                 single_size += sizeof(bool);
                 break;
-            case meta::DataType::DOUBLE:
+            case DataType::DOUBLE:
                 single_size += sizeof(double);
                 break;
-            case meta::DataType::FLOAT:
+            case DataType::FLOAT:
                 single_size += sizeof(float);
                 break;
-            case meta::DataType::INT8:
+            case DataType::INT8:
                 single_size += sizeof(uint8_t);
                 break;
-            case meta::DataType::INT16:
+            case DataType::INT16:
                 single_size += sizeof(uint16_t);
                 break;
-            case meta::DataType::INT32:
+            case DataType::INT32:
                 single_size += sizeof(uint32_t);
                 break;
-            case meta::DataType::UID:
-            case meta::DataType::INT64:
+            case DataType::UID:
+            case DataType::INT64:
                 single_size += sizeof(uint64_t);
                 break;
-            case meta::DataType::VECTOR_FLOAT:
-            case meta::DataType::VECTOR_BINARY: {
+            case DataType::VECTOR_FLOAT:
+            case DataType::VECTOR_BINARY: {
                 json params = field->GetParams();
                 if (params.find(knowhere::meta::DIM) == params.end()) {
                     std::string msg = "Vector field params must contain: dimension";
@@ -160,7 +159,7 @@ MemSegment::GetSingleEntitySize(int64_t& single_size) {
                 }
 
                 int64_t dimension = params[knowhere::meta::DIM];
-                if (ftype == meta::DataType::VECTOR_BINARY) {
+                if (ftype == DataType::VECTOR_BINARY) {
                     single_size += (dimension / 8);
                 } else {
                     single_size += (dimension * sizeof(float));

core/src/db/merge/MergeTask.h

@@ -13,8 +13,8 @@
 
 #include <string>
 
+#include "db/Types.h"
 #include "db/merge/MergeManager.h"
-#include "db/meta/MetaTypes.h"
 #include "db/snapshot/ResourceTypes.h"
 #include "db/snapshot/Snapshot.h"
 #include "utils/Status.h"

core/src/db/meta/MetaConsts.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#pragma once
-
-namespace milvus {
-namespace engine {
-namespace meta {
-
-const size_t S_PS = 1UL;
-const size_t MS_PS = 1000 * S_PS;
-const size_t US_PS = 1000 * MS_PS;
-const size_t NS_PS = 1000 * US_PS;
-
-const size_t SECOND = 1UL;
-const size_t MINUTE = 60 * SECOND;
-const size_t HOUR = 60 * MINUTE;
-const size_t DAY = 24 * HOUR;
-const size_t WEEK = 7 * DAY;
-
-// This value is to ignore small raw files when building index.
-// The reason is:
-// 1. The performance of brute-search for small raw files could be better than small index file.
-// 2. And small raw files can be merged to larger files, thus reduce fragmented files count.
-// We decide the value based on a testing for small size raw/index files.
-const size_t BUILD_INDEX_THRESHOLD = 4096;
-
-}  // namespace meta
-}  // namespace engine
-}  // namespace milvus

core/src/db/meta/MetaTypes.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#pragma once
-
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "db/Constants.h"
-#include "knowhere/index/IndexType.h"
-#include "src/version.h"
-
-namespace milvus {
-namespace engine {
-namespace meta {
-
-using DateT = int;
-
-enum DataType {
-    NONE = 0,
-    BOOL = 1,
-    INT8 = 2,
-    INT16 = 3,
-    INT32 = 4,
-    INT64 = 5,
-
-    FLOAT = 10,
-    DOUBLE = 11,
-
-    STRING = 20,
-
-    UID = 30,
-
-    VECTOR_BINARY = 100,
-    VECTOR_FLOAT = 101,
-};
-
-}  // namespace meta
-}  // namespace engine
-}  // namespace milvus

core/src/db/snapshot/Context.h

@@ -16,7 +16,8 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include "db/meta/MetaTypes.h"
+
+#include "db/Types.h"
 #include "db/snapshot/Resources.h"
 #include "db/snapshot/Snapshot.h"
 

core/src/db/snapshot/ResourceTypes.h

@@ -15,7 +15,7 @@
 #include <string>
 #include <vector>
 
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 
 namespace milvus {
 namespace engine {

core/src/db/snapshot/Store.h

@@ -284,7 +284,7 @@ class Store : public std::enable_shared_from_this<Store> {
                 std::stringstream fname;
                 fname << "f_" << fi << "_" << ++id_map[FieldElement::Name];
 
-                Field temp_f(fname.str(), fi, FieldType::VECTOR_FLOAT);
+                Field temp_f(fname.str(), fi, DataType::VECTOR_FLOAT);
                 FieldPtr field;
 
                 temp_f.Activate();

core/src/db/wal/WalDefinations.h

@@ -17,7 +17,6 @@
 #include <vector>
 
 #include "db/Types.h"
-#include "db/meta/MetaTypes.h"
 #include "segment/Segment.h"
 
 namespace milvus {

core/src/db/wal/WalMetaHandler.h

@@ -15,8 +15,8 @@
 #include <string>
 #include <unordered_map>
 
+#include "db/Types.h"
 #include "db/meta/MetaFactory.h"
-#include "db/meta/MetaTypes.h"
 #include "db/wal/WalDefinations.h"
 #include "db/wal/WalFileHandler.h"
 

core/src/metrics/Metrics.h

@@ -12,7 +12,7 @@
 #pragma once
 
 #include "MetricBase.h"
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 
 namespace milvus {
 namespace server {

core/src/query/GeneralQuery.h

@@ -17,7 +17,8 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
-#include "db/meta/MetaTypes.h"
+
+#include "db/Types.h"
 #include "utils/Json.h"
 
 namespace milvus {

core/src/scheduler/Definition.h

@@ -21,16 +21,16 @@
 #include <unordered_map>
 #include <vector>
 
+#include "db/Types.h"
 #include "db/engine/EngineFactory.h"
 #include "db/engine/ExecutionEngine.h"
-#include "db/meta/MetaTypes.h"
 
 namespace milvus {
 namespace scheduler {
 
 using ExecutionEnginePtr = engine::ExecutionEnginePtr;
 using EngineFactory = engine::EngineFactory;
-using DataType = engine::meta::DataType;
+using DataType = engine::DataType;
 
 constexpr uint64_t TASK_TABLE_MAX_COUNT = 1ULL << 16ULL;
 

core/src/segment/Segment.h

@@ -22,7 +22,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "db/meta/MetaTypes.h"
+#include "db/Types.h"
 #include "knowhere/index/vector_index/VecIndex.h"
 #include "segment/DeletedDocs.h"
 #include "segment/IdBloomFilter.h"
@@ -32,8 +32,8 @@ namespace engine {
 
 extern const char* COLLECTIONS_FOLDER;
 
-using FIELD_TYPE = engine::meta::DataType;
-using FIELD_TYPE_MAP = std::unordered_map<std::string, engine::meta::DataType>;
+using FIELD_TYPE = engine::DataType;
+using FIELD_TYPE_MAP = std::unordered_map<std::string, FIELD_TYPE>;
 using FIELD_WIDTH_MAP = std::unordered_map<std::string, int64_t>;
 using FIXED_FIELD_DATA = std::vector<uint8_t>;
 using FIXEDX_FIELD_MAP = std::unordered_map<std::string, FIXED_FIELD_DATA>;

core/src/segment/SegmentWriter.h

@@ -17,13 +17,13 @@
 
 #pragma once
 
-#include <src/db/meta/MetaTypes.h>
 #include <memory>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
 #include "db/SnapshotVisitor.h"
+#include "db/Types.h"
 #include "segment/Segment.h"
 #include "segment/SegmentReader.h"
 #include "storage/FSHandler.h"

core/src/server/ValidationUtil.h

@@ -12,7 +12,6 @@
 #pragma once
 
 #include "db/Types.h"
-#include "db/meta/MetaTypes.h"
 #include "utils/Json.h"
 #include "utils/Status.h"
 

core/src/server/delivery/request/BaseReq.h

@@ -12,7 +12,6 @@
 #pragma once
 
 #include "db/Types.h"
-#include "db/meta/MetaTypes.h"
 #include "grpc/gen-milvus/milvus.grpc.pb.h"
 #include "grpc/gen-status/status.grpc.pb.h"
 #include "grpc/gen-status/status.pb.h"
@@ -33,7 +32,7 @@ namespace milvus {
 namespace server {
 
 struct FieldSchema {
-    engine::FieldType field_type_;
+    engine::DataType field_type_;
     milvus::json field_params_;
     milvus::json index_params_;
 };

core/src/server/delivery/request/CmdReq.cpp

@@ -13,6 +13,7 @@
 #include "config/ConfigMgr.h"
 #include "metrics/SystemInfo.h"
 #include "scheduler/SchedInst.h"
+#include "src/version.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 

core/src/server/delivery/request/CreateCollectionReq.cpp

@@ -79,12 +79,12 @@ CreateCollectionReq::OnExecute() {
                 index_name = index_params["name"];
             }
 
-            if (field_type == engine::FieldType::VECTOR_FLOAT || field_type == engine::FieldType::VECTOR_BINARY) {
+            if (field_type == engine::DataType::VECTOR_FLOAT || field_type == engine::DataType::VECTOR_BINARY) {
                 if (!field_params.contains(engine::PARAM_DIMENSION)) {
                     return Status(SERVER_INVALID_VECTOR_DIMENSION, "Dimension not defined in field_params");
                 } else {
                     auto dim = field_params[engine::PARAM_DIMENSION].get<int64_t>();
-                    if (field_type == engine::FieldType::VECTOR_FLOAT) {
+                    if (field_type == engine::DataType::VECTOR_FLOAT) {
                         STATUS_CHECK(ValidateDimension(dim, false));
                     } else {
                         STATUS_CHECK(ValidateDimension(dim, true));

core/src/server/delivery/request/GetCollectionInfoReq.cpp

@@ -51,7 +51,7 @@ GetCollectionInfoReq::OnExecute() {
         collection_schema_.extra_params_ = collection->GetParams();
         for (auto& field_kv : collection_mappings) {
             auto field = field_kv.first;
-            if (field->GetFtype() == (engine::snapshot::FTYPE_TYPE)engine::meta::DataType::UID) {
+            if (field->GetFtype() == (engine::snapshot::FTYPE_TYPE)engine::DataType::UID) {
                 continue;
             }
 
@@ -66,7 +66,7 @@ GetCollectionInfoReq::OnExecute() {
 
             auto field_name = field->GetName();
             FieldSchema field_schema;
-            field_schema.field_type_ = (engine::FieldType)field->GetFtype();
+            field_schema.field_type_ = (engine::DataType)field->GetFtype();
             field_schema.field_params_ = field->GetParams();
             field_schema.index_params_ = field_index_param;
 

core/src/server/delivery/request/GetEntityByIDReq.cpp

@@ -16,7 +16,8 @@
 // under the License.
 
 #include "server/delivery/request/GetEntityByIDReq.h"
-#include "db/meta/MetaTypes.h"
+
+#include "db/Types.h"
 #include "server/DBWrapper.h"
 #include "server/ValidationUtil.h"
 #include "utils/TimeRecorder.h"

core/src/server/delivery/request/SearchReq.cpp

@@ -66,12 +66,12 @@ SearchReq::OnExecute() {
         int64_t dimension = 0;
 
         // step 4: Get field info
-        std::unordered_map<std::string, engine::meta::DataType> field_types;
+        std::unordered_map<std::string, engine::DataType> field_types;
         for (auto& schema : fields_schema) {
             auto field = schema.first;
-            field_types.insert(std::make_pair(field->GetName(), (engine::meta::DataType)field->GetFtype()));
-            if (field->GetFtype() == (int)engine::meta::DataType::VECTOR_FLOAT ||
-                field->GetFtype() == (int)engine::meta::DataType::VECTOR_BINARY) {
+            field_types.insert(std::make_pair(field->GetName(), (engine::DataType)field->GetFtype()));
+            if (field->GetFtype() == (int)engine::DataType::VECTOR_FLOAT ||
+                field->GetFtype() == (int)engine::DataType::VECTOR_BINARY) {
                 dimension = field->GetParams()[engine::PARAM_DIMENSION];
             }
         }

core/src/server/grpc_impl/GrpcRequestHandler.cpp

@@ -310,7 +310,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                     float grpc_float_data;
                     double grpc_double_data;
                     switch (attr.attr_type_.at(field_name)) {
-                        case engine::meta::DataType::INT8: {
+                        case engine::DataType::INT8: {
                             if (attr_data.size() == sizeof(int8_t)) {
                                 grpc_int32_data = attr_data[0];
                                 int32_data.emplace_back(grpc_int32_data);
@@ -320,7 +320,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                             }
                             break;
                         }
-                        case engine::meta::DataType::INT16: {
+                        case engine::DataType::INT16: {
                             if (attr_data.size() == sizeof(int16_t)) {
                                 int16_t value;
                                 memcpy(&value, attr_data.data(), sizeof(int16_t));
@@ -332,7 +332,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                             }
                             break;
                         }
-                        case engine::meta::DataType::INT32: {
+                        case engine::DataType::INT32: {
                             if (attr_data.size() == sizeof(int32_t)) {
                                 memcpy(&grpc_int32_data, attr_data.data(), sizeof(int32_t));
                                 int32_data.emplace_back(grpc_int32_data);
@@ -342,7 +342,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                             }
                             break;
                         }
-                        case engine::meta::DataType::INT64: {
+                        case engine::DataType::INT64: {
                             if (attr_data.size() == sizeof(int64_t)) {
                                 memcpy(&grpc_int64_data, attr_data.data(), sizeof(int64_t));
                                 int64_data.emplace_back(grpc_int64_data);
@@ -352,7 +352,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                             }
                             break;
                         }
-                        case engine::meta::DataType::FLOAT: {
+                        case engine::DataType::FLOAT: {
                             if (attr_data.size() == sizeof(float)) {
                                 float value;
                                 memcpy(&value, attr_data.data(), sizeof(float));
@@ -364,7 +364,7 @@ ConstructEntityResults(const std::vector<engine::AttrsData>& attrs, const std::v
                             }
                             break;
                         }
-                        case engine::meta::DataType::DOUBLE: {
+                        case engine::DataType::DOUBLE: {
                             if (attr_data.size() == sizeof(double)) {
                                 memcpy(&grpc_double_data, attr_data.data(), sizeof(double));
                                 double_data.emplace_back(grpc_double_data);
@@ -639,7 +639,7 @@ GrpcRequestHandler::CreateCollection(::grpc::ServerContext* context, const ::mil
         const auto& field = request->fields(i);
 
         FieldSchema field_schema;
-        field_schema.field_type_ = (engine::FieldType)field.type();
+        field_schema.field_type_ = (engine::DataType)field.type();
 
         // Currently only one extra_param
         if (field.extra_params_size() != 0) {
@@ -762,7 +762,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
 
         auto single_size = data.size() / id_size;
 
-        if (type == engine::meta::DataType::UID) {
+        if (type == engine::DataType::UID) {
             int64_t int64_value;
             auto int64_size = single_size * sizeof(int8_t) / sizeof(int64_t);
             for (int i = 0; i < id_size; i++) {
@@ -779,7 +779,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
         field_value->set_field_name(name);
         field_value->set_type(static_cast<milvus::grpc::DataType>(type));
         // general data
-        if (type == engine::meta::DataType::VECTOR_BINARY) {
+        if (type == engine::DataType::VECTOR_BINARY) {
             // add binary vector data
             std::vector<int8_t> binary_vector;
             auto vector_size = single_size * sizeof(int8_t) / sizeof(int8_t);
@@ -792,7 +792,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
                 memcpy(vector_row_record->mutable_binary_data()->data(), binary_vector.data(), binary_vector.size());
             }
 
-        } else if (type == engine::meta::DataType::VECTOR_FLOAT) {
+        } else if (type == engine::DataType::VECTOR_FLOAT) {
             // add float vector data
             std::vector<float> float_vector;
             auto vector_size = single_size * sizeof(int8_t) / sizeof(float);
@@ -808,7 +808,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
         } else {
             // add attribute data
             auto attr_record = field_value->mutable_attr_record();
-            if (type == engine::meta::DataType::INT32) {
+            if (type == engine::DataType::INT32) {
                 // add int32 data
                 int32_t int32_value;
                 auto int32_size = single_size * sizeof(int8_t) / sizeof(int32_t);
@@ -817,7 +817,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
                     memcpy(&int32_value, data.data() + offset, single_size);
                     attr_record->add_int32_value(int32_value);
                 }
-            } else if (type == engine::meta::DataType::INT64) {
+            } else if (type == engine::DataType::INT64) {
                 // add int64 data
                 int64_t int64_value;
                 auto int64_size = single_size * sizeof(int8_t) / sizeof(int64_t);
@@ -826,7 +826,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
                     memcpy(&int64_value, data.data() + offset, single_size);
                     attr_record->add_int64_value(int64_value);
                 }
-            } else if (type == engine::meta::DataType::DOUBLE) {
+            } else if (type == engine::DataType::DOUBLE) {
                 // add double data
                 double double_value;
                 auto int32_size = single_size * sizeof(int8_t) / sizeof(double);
@@ -835,7 +835,7 @@ GrpcRequestHandler::GetEntityByID(::grpc::ServerContext* context, const ::milvus
                     memcpy(&double_value, data.data() + offset, single_size);
                     attr_record->add_double_value(double_value);
                 }
-            } else if (type == engine::meta::DataType::FLOAT) {
+            } else if (type == engine::DataType::FLOAT) {
                 // add float data
                 float float_value;
                 auto float_size = single_size * sizeof(int8_t) / sizeof(float);
@@ -1416,7 +1416,7 @@ GrpcRequestHandler::SearchPB(::grpc::ServerContext* context, const ::milvus::grp
 #if 0
 Status
 ParseTermQuery(const nlohmann::json& term_json,
-               std::unordered_map<std::string, engine::meta::DataType> field_type,
+               std::unordered_map<std::string, engine::DataType> field_type,
                query::TermQueryPtr& term_query) {
     std::string field_name = term_json["field"].get<std::string>();
     auto term_value_json = term_json["values"];
@@ -1430,7 +1430,7 @@ ParseTermQuery(const nlohmann::json& term_json,
     term_query->field_value.resize(term_size * sizeof(int64_t));
 
     switch (field_type.at(field_name)) {
-        case engine::meta::DataType::INT8: {
+        case engine::DataType::INT8: {
             std::vector<int64_t> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; i++) {
                 term_value[i] = term_value_json[i].get<int8_t>();
@@ -1438,7 +1438,7 @@ ParseTermQuery(const nlohmann::json& term_json,
             memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(int64_t));
             break;
         }
-        case engine::meta::DataType::INT16: {
+        case engine::DataType::INT16: {
             std::vector<int64_t> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; i++) {
                 term_value[i] = term_value_json[i].get<int16_t>();
@@ -1446,7 +1446,7 @@ ParseTermQuery(const nlohmann::json& term_json,
             memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(int64_t));
             break;
         }
-        case engine::meta::DataType::INT32: {
+        case engine::DataType::INT32: {
             std::vector<int64_t> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; i++) {
                 term_value[i] = term_value_json[i].get<int32_t>();
@@ -1454,7 +1454,7 @@ ParseTermQuery(const nlohmann::json& term_json,
             memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(int64_t));
             break;
         }
-        case engine::meta::DataType::INT64: {
+        case engine::DataType::INT64: {
             std::vector<int64_t> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; ++i) {
                 term_value[i] = term_value_json[i].get<int64_t>();
@@ -1462,7 +1462,7 @@ ParseTermQuery(const nlohmann::json& term_json,
             memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(int64_t));
             break;
         }
-        case engine::meta::DataType::FLOAT: {
+        case engine::DataType::FLOAT: {
             std::vector<double> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; ++i) {
                 term_value[i] = term_value_json[i].get<float>();
@@ -1470,7 +1470,7 @@ ParseTermQuery(const nlohmann::json& term_json,
             memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(double));
             break;
         }
-        case engine::meta::DataType::DOUBLE: {
+        case engine::DataType::DOUBLE: {
             std::vector<double> term_value(term_size, 0);
             for (uint64_t i = 0; i < term_size; ++i) {
                 term_value[i] = term_value_json[i].get<double>();

core/src/server/web_impl/handler/WebRequestHandler.cpp

@@ -423,10 +423,10 @@ WebRequestHandler::ProcessLeafQueryJson(const nlohmann::json& json, milvus::quer
         //        term_query->field_value.resize(term_size * sizeof(int64_t));
         //
         //        switch (field_type_.at(field_name)) {
-        //            case engine::meta::DataType::INT8:
-        //            case engine::meta::DataType::INT16:
-        //            case engine::meta::DataType::INT32:
-        //            case engine::meta::DataType::INT64: {
+        //            case engine::DataType::INT8:
+        //            case engine::DataType::INT16:
+        //            case engine::DataType::INT32:
+        //            case engine::DataType::INT64: {
         //                std::vector<int64_t> term_value(term_size, 0);
         //                for (uint64_t i = 0; i < term_size; ++i) {
         //                    term_value[i] = term_value_json[i].get<int64_t>();
@@ -434,8 +434,8 @@ WebRequestHandler::ProcessLeafQueryJson(const nlohmann::json& json, milvus::quer
         //                memcpy(term_query->field_value.data(), term_value.data(), term_size * sizeof(int64_t));
         //                break;
         //            }
-        //            case engine::meta::DataType::FLOAT:
-        //            case engine::meta::DataType::DOUBLE: {
+        //            case engine::DataType::FLOAT:
+        //            case engine::DataType::DOUBLE: {
         //                std::vector<double> term_value(term_size, 0);
         //                for (uint64_t i = 0; i < term_size; ++i) {
         //                    term_value[i] = term_value_json[i].get<double>();
@@ -606,35 +606,35 @@ ConvertRowToColumnJson(const std::vector<engine::AttrsData>& row_attrs, const st
             double double_value;
             auto attr_data = attr.attr_data_.at(field_names[i]);
             switch (attr.attr_type_.at(field_names[i])) {
-                case engine::meta::DataType::INT8: {
+                case engine::DataType::INT8: {
                     if (attr_data.size() == sizeof(int8_t)) {
                         int_value = attr_data[0];
                         int_data.emplace_back(int_value);
                     }
                     break;
                 }
-                case engine::meta::DataType::INT16: {
+                case engine::DataType::INT16: {
                     if (attr_data.size() == sizeof(int16_t)) {
                         memcpy(&int_value, attr_data.data(), sizeof(int16_t));
                         int_data.emplace_back(int_value);
                     }
                     break;
                 }
-                case engine::meta::DataType::INT32: {
+                case engine::DataType::INT32: {
                     if (attr_data.size() == sizeof(int32_t)) {
                         memcpy(&int_value, attr_data.data(), sizeof(int32_t));
                         int_data.emplace_back(int_value);
                     }
                     break;
                 }
-                case engine::meta::DataType::INT64: {
+                case engine::DataType::INT64: {
                     if (attr_data.size() == sizeof(int64_t)) {
                         memcpy(&int_value, attr_data.data(), sizeof(int64_t));
                         int_data.emplace_back(int_value);
                     }
                     break;
                 }
-                case engine::meta::DataType::FLOAT: {
+                case engine::DataType::FLOAT: {
                     if (attr_data.size() == sizeof(float)) {
                         float float_value;
                         memcpy(&float_value, attr_data.data(), sizeof(float));
@@ -643,7 +643,7 @@ ConvertRowToColumnJson(const std::vector<engine::AttrsData>& row_attrs, const st
                     }
                     break;
                 }
-                case engine::meta::DataType::DOUBLE: {
+                case engine::DataType::DOUBLE: {
                     if (attr_data.size() == sizeof(double)) {
                         memcpy(&double_value, attr_data.data(), sizeof(double));
                         double_data.emplace_back(double_value);
@@ -819,19 +819,19 @@ WebRequestHandler::GetEntityByIDs(const std::string& collection_name, const std:
         std::string name = it.first->GetName();
         uint64_t type = it.first->GetFtype();
         std::vector<uint8_t> data = data_chunk->fixed_fields_[name];
-        if (type == engine::FieldType::VECTOR_BINARY) {
+        if (type == engine::DataType::VECTOR_BINARY) {
             engine::VectorsData vectors_data;
             memcpy(vectors_data.binary_data_.data(), data.data(), data.size());
             memcpy(vectors_data.id_array_.data(), id_array.data(), id_array.size());
             vector_batch.emplace_back(vectors_data);
-        } else if (type == engine::FieldType::VECTOR_FLOAT) {
+        } else if (type == engine::DataType::VECTOR_FLOAT) {
             engine::VectorsData vectors_data;
             memcpy(vectors_data.float_data_.data(), data.data(), data.size());
             memcpy(vectors_data.id_array_.data(), id_array.data(), id_array.size());
             vector_batch.emplace_back(vectors_data);
         } else {
             engine::AttrsData attrs_data;
-            attrs_data.attr_type_[name] = static_cast<engine::meta::DataType>(type);
+            attrs_data.attr_type_[name] = static_cast<engine::DataType>(type);
             attrs_data.attr_data_[name] = data;
             memcpy(attrs_data.id_array_.data(), id_array.data(), id_array.size());
             attr_batch.emplace_back(attrs_data);
@@ -1209,17 +1209,17 @@ WebRequestHandler::CreateHybridCollection(const milvus::server::web::OString& bo
 
         const std::string& field_type = field["field_type"];
         if (field_type == "int8") {
-            field_schema.field_type_ = engine::FieldType::INT8;
+            field_schema.field_type_ = engine::DataType::INT8;
         } else if (field_type == "int16") {
-            field_schema.field_type_ = engine::FieldType::INT16;
+            field_schema.field_type_ = engine::DataType::INT16;
         } else if (field_type == "int32") {
-            field_schema.field_type_ = engine::FieldType::INT32;
+            field_schema.field_type_ = engine::DataType::INT32;
         } else if (field_type == "int64") {
-            field_schema.field_type_ = engine::FieldType::INT64;
+            field_schema.field_type_ = engine::DataType::INT64;
         } else if (field_type == "float") {
-            field_schema.field_type_ = engine::FieldType::FLOAT;
+            field_schema.field_type_ = engine::DataType::FLOAT;
         } else if (field_type == "double") {
-            field_schema.field_type_ = engine::FieldType::DOUBLE;
+            field_schema.field_type_ = engine::DataType::DOUBLE;
         } else if (field_type == "vector") {
         } else {
             std::string msg = field_name + " has wrong field_type";
@@ -1582,7 +1582,7 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
     std::string partition_name = body_json["partition_tag"];
     int32_t row_num = body_json["row_num"];
 
-    std::unordered_map<std::string, engine::meta::DataType> field_types;
+    std::unordered_map<std::string, engine::DataType> field_types;
     auto status = Status::OK();
     // auto status = req_handler_.DescribeHybridCollection(context_ptr_, collection_name->c_str(), field_types);
 
@@ -1603,23 +1603,23 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
 
         std::vector<uint8_t> temp_data;
         switch (field_types.at(field_name)) {
-            case engine::meta::DataType::INT32: {
+            case engine::DataType::INT32: {
                 CopyStructuredData<int32_t>(field_value, temp_data);
                 break;
             }
-            case engine::meta::DataType::INT64: {
+            case engine::DataType::INT64: {
                 CopyStructuredData<int64_t>(field_value, temp_data);
                 break;
             }
-            case engine::meta::DataType::FLOAT: {
+            case engine::DataType::FLOAT: {
                 CopyStructuredData<float>(field_value, temp_data);
                 break;
             }
-            case engine::meta::DataType::DOUBLE: {
+            case engine::DataType::DOUBLE: {
                 CopyStructuredData<double>(field_value, temp_data);
                 break;
             }
-            case engine::meta::DataType::VECTOR_FLOAT: {
+            case engine::DataType::VECTOR_FLOAT: {
                 bool bin_flag;
                 status = IsBinaryCollection(collection_name->c_str(), bin_flag);
                 if (!status.ok()) {

core/src/server/web_impl/handler/WebRequestHandler.h

@@ -243,7 +243,7 @@ class WebRequestHandler {
     std::shared_ptr<Context> context_ptr_;
     ReqHandler req_handler_;
     query::QueryPtr query_ptr_;
-    std::unordered_map<std::string, engine::meta::DataType> field_type_;
+    std::unordered_map<std::string, engine::DataType> field_type_;
 };
 
 }  // namespace web

core/unittest/db/test_db.cpp

@@ -35,11 +35,11 @@ CreateCollection(std::shared_ptr<DBImpl> db, const std::string& collection_name,
     auto collection_schema = std::make_shared<Collection>(collection_name);
     context.collection = collection_schema;
     auto vector_field = std::make_shared<Field>("vector", 0,
-                                                milvus::engine::FieldType::VECTOR_FLOAT);
+                                                milvus::engine::DataType::VECTOR_FLOAT);
     auto vector_field_element = std::make_shared<FieldElement>(0, 0, "ivfsq8",
                                                                milvus::engine::FieldElementType::FET_INDEX);
     auto int_field = std::make_shared<Field>("int", 0,
-                                             milvus::engine::FieldType::INT32);
+                                             milvus::engine::DataType::INT32);
     context.fields_schema[vector_field] = {vector_field_element};
     context.fields_schema[int_field] = {};
 
@@ -57,13 +57,13 @@ CreateCollection2(std::shared_ptr<DBImpl> db, const std::string& collection_name
 
     milvus::json params;
     params[milvus::knowhere::meta::DIM] = COLLECTION_DIM;
-    auto vector_field = std::make_shared<Field>("vector", 0, milvus::engine::FieldType::VECTOR_FLOAT, params);
+    auto vector_field = std::make_shared<Field>("vector", 0, milvus::engine::DataType::VECTOR_FLOAT, params);
     context.fields_schema[vector_field] = {};
 
-    std::unordered_map<std::string, milvus::engine::meta::DataType> attr_type = {
-        {"field_0", milvus::engine::FieldType::INT32},
-        {"field_1", milvus::engine::FieldType::INT64},
-        {"field_2", milvus::engine::FieldType::DOUBLE},
+    std::unordered_map<std::string, milvus::engine::DataType> attr_type = {
+        {"field_0", milvus::engine::DataType::INT32},
+        {"field_1", milvus::engine::DataType::INT64},
+        {"field_2", milvus::engine::DataType::DOUBLE},
     };
 
     std::vector<std::string> field_names;
@@ -380,7 +380,7 @@ TEST_F(DBTest, QueryTest) {
     milvus::query::GeneralQueryPtr general_query;
     milvus::query::QueryPtr query_ptr;
     std::vector<std::string> field_names;
-    std::unordered_map<std::string, milvus::engine::meta::DataType> attr_type;
+    std::unordered_map<std::string, milvus::engine::DataType> attr_type;
     milvus::engine::QueryResult result;
     //db_->Query(ctx1, c1, partition_patterns, general_query, query_ptr, field_names, attr_type, result);
 }
@@ -562,6 +562,6 @@ TEST_F(DBTest, StatsTest) {
     int64_t row_count = json_stats[milvus::engine::JSON_ROW_COUNT];
     ASSERT_EQ(row_count, entity_count * 2);
 
-//    std::string ss = json_stats.dump();
-//    std::cout << ss << std::endl;
+    std::string ss = json_stats.dump();
+    std::cout << ss << std::endl;
 }

core/unittest/db/test_segment.cpp

@@ -39,7 +39,7 @@ CreateCollection(std::shared_ptr<DBImpl> db, const std::string& collection_name,
     int64_t field_id = 0;
     /* field uid */
     auto uid_field = std::make_shared<Field>(milvus::engine::DEFAULT_UID_NAME, 0,
-            milvus::engine::FieldType::UID, milvus::engine::snapshot::JEmpty, field_id);
+            milvus::engine::DataType::UID, milvus::engine::snapshot::JEmpty, field_id);
     auto uid_field_element_blt = std::make_shared<FieldElement>(collection_id, field_id,
             milvus::engine::DEFAULT_BLOOM_FILTER_NAME, milvus::engine::FieldElementType::FET_BLOOM_FILTER);
     auto uid_field_element_del = std::make_shared<FieldElement>(collection_id, field_id,
@@ -48,7 +48,7 @@ CreateCollection(std::shared_ptr<DBImpl> db, const std::string& collection_name,
     field_id++;
     /* field vector */
     milvus::json vector_param = {{milvus::knowhere::meta::DIM, 4}};
-    auto vector_field = std::make_shared<Field>("vector", 0, milvus::engine::FieldType::VECTOR_FLOAT, vector_param,
+    auto vector_field = std::make_shared<Field>("vector", 0, milvus::engine::DataType::VECTOR_FLOAT, vector_param,
             field_id);
     auto vector_field_element_index = std::make_shared<FieldElement>(collection_id, field_id,
             milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, milvus::engine::FieldElementType::FET_INDEX);

core/unittest/db/test_ss_event.cpp

@@ -23,7 +23,7 @@ using SegmentCommitPtr = milvus::engine::snapshot::SegmentCommitPtr;
 using SchemaCommit = milvus::engine::snapshot::SchemaCommit;
 using FieldCommit = milvus::engine::snapshot::FieldCommit;
 
-using FType = milvus::engine::FieldType;
+using FType = milvus::engine::DataType;
 using FEType = milvus::engine::FieldElementType;
 
 using InActiveResourcesGCEvent = milvus::engine::snapshot::InActiveResourcesGCEvent;

core/unittest/db/test_ss_meta.cpp

@@ -19,7 +19,7 @@ using ResourceContext = milvus::engine::snapshot::ResourceContext<T>;
 template <typename T>
 using ResourceContextBuilder = milvus::engine::snapshot::ResourceContextBuilder<T>;
 
-using FType = milvus::engine::FieldType;
+using FType = milvus::engine::DataType;
 using FEType = milvus::engine::FieldElementType;
 using Op = milvus::engine::meta::MetaContextOp;
 

core/unittest/db/utils.h

@@ -215,11 +215,11 @@ CreateCollection(const std::string& collection_name, const LSN_TYPE& lsn) {
     auto collection_schema = std::make_shared<Collection>(collection_name);
     context.collection = collection_schema;
     auto vector_field = std::make_shared<Field>("vector", 0,
-            milvus::engine::FieldType::VECTOR_FLOAT);
+            milvus::engine::DataType::VECTOR_FLOAT);
     auto vector_field_element = std::make_shared<FieldElement>(0, 0, "ivfsq8",
             milvus::engine::FieldElementType::FET_INDEX);
     auto int_field = std::make_shared<Field>("int", 0,
-            milvus::engine::FieldType::INT32);
+            milvus::engine::DataType::INT32);
     context.fields_schema[vector_field] = {vector_field_element};
     context.fields_schema[int_field] = {};