未验证 提交 1a5fad87 编写于 作者: F FluorineDog 提交者: GitHub

Split Ftype into Ftype and FEtype (#3341)

* fix include directories not find bug (#3323)
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>
Co-authored-by: Nyangxuan <xuan.yang@zilliz.com>
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* reduce grpc download file size from 380M to 130M  (#3326)

* reduce grpc download file size from 380M to 130M
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* fix bug
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>
Co-authored-by: Nyangxuan <xuan.yang@zilliz.com>
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* update clang-tidy config (#3314)

* update clang-tidy config
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update .clang-tidy
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update clang-tidy config
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>

* update clang-tidy HeaderFilterRegex
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* enable fetype
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* rename field_element_method
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* add FETYPE_TYPE
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* Add WebServer unittest (#3321)

* Add web server interface
Signed-off-by: Nfishpenguin <kun.yu@zilliz.com>

* Add unittest/server
Signed-off-by: Nfishpenguin <kun.yu@zilliz.com>

* Add web server ut
Signed-off-by: Nfishpenguin <kun.yu@zilliz.com>
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* modify db schema and remove conversion
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* weak fetype
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* enforce strong fetype
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* lint
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* format code
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>

* [skip ci] fix
Signed-off-by: Nfluorinedog <fluorinedog@gmail.com>
Co-authored-by: NXuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com>
Co-authored-by: Nyangxuan <xuan.yang@zilliz.com>
Co-authored-by: NCai Yudong <yudong.cai@zilliz.com>
Co-authored-by: Nyukun <kun.yu@zilliz.com>
上级 79709ddc
...@@ -92,7 +92,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na ...@@ -92,7 +92,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
auto field_elements = ss->GetFieldElementsByField(field_name); auto field_elements = ss->GetFieldElementsByField(field_name);
if (IsVectorField(field)) { if (IsVectorField(field)) {
for (auto& field_element : field_elements) { for (auto& field_element : field_elements) {
if (field_element->GetFtype() == static_cast<snapshot::FTYPE_TYPE>(engine::FieldElementType::FET_INDEX)) { if (field_element->GetFEtype() == engine::FieldElementType::FET_INDEX) {
index_info.index_name_ = field_element->GetName(); index_info.index_name_ = field_element->GetName();
index_info.index_type_ = field_element->GetTypeName(); index_info.index_type_ = field_element->GetTypeName();
auto json = field_element->GetParams(); auto json = field_element->GetParams();
...@@ -107,7 +107,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na ...@@ -107,7 +107,7 @@ GetSnapshotIndex(const std::string& collection_name, const std::string& field_na
} }
} else { } else {
for (auto& field_element : field_elements) { for (auto& field_element : field_elements) {
if (field_element->GetFtype() == static_cast<snapshot::FTYPE_TYPE>(engine::FieldElementType::FET_INDEX)) { if (field_element->GetFEtype() == engine::FieldElementType::FET_INDEX) {
index_info.index_name_ = field_element->GetName(); index_info.index_name_ = field_element->GetName();
index_info.index_type_ = field_element->GetTypeName(); index_info.index_type_ = field_element->GetTypeName();
} }
...@@ -134,8 +134,8 @@ DeleteSnapshotIndex(const std::string& collection_name, const std::string& field ...@@ -134,8 +134,8 @@ DeleteSnapshotIndex(const std::string& collection_name, const std::string& field
STATUS_CHECK(snapshot::Snapshots::GetInstance().GetSnapshot(ss, collection_name)); STATUS_CHECK(snapshot::Snapshots::GetInstance().GetSnapshot(ss, collection_name));
std::vector<snapshot::FieldElementPtr> elements = ss->GetFieldElementsByField(name); std::vector<snapshot::FieldElementPtr> elements = ss->GetFieldElementsByField(name);
for (auto& element : elements) { for (auto& element : elements) {
if (element->GetFtype() == engine::FieldElementType::FET_INDEX || if (element->GetFEtype() == engine::FieldElementType::FET_INDEX ||
element->GetFtype() == engine::FieldElementType::FET_COMPRESS_SQ8) { element->GetFEtype() == engine::FieldElementType::FET_COMPRESS_SQ8) {
snapshot::OperationContext context; snapshot::OperationContext context;
context.stale_field_elements.push_back(element); context.stale_field_elements.push_back(element);
auto op = std::make_shared<snapshot::DropAllIndexOperation>(context, ss); auto op = std::make_shared<snapshot::DropAllIndexOperation>(context, ss);
...@@ -225,7 +225,7 @@ GetSnapshotInfo(const std::string& collection_name, milvus::json& json_info) { ...@@ -225,7 +225,7 @@ GetSnapshotInfo(const std::string& collection_name, milvus::json& json_info) {
// if the element is index, print index name/type // if the element is index, print index name/type
// else print element name // else print element name
if (element->GetFtype() == engine::FieldElementType::FET_INDEX) { if (element->GetFEtype() == engine::FieldElementType::FET_INDEX) {
json_file[JSON_NAME] = element->GetName(); json_file[JSON_NAME] = element->GetName();
json_file[JSON_INDEX_TYPE] = element->GetTypeName(); json_file[JSON_INDEX_TYPE] = element->GetTypeName();
} else { } else {
......
...@@ -113,7 +113,7 @@ class SegmentFieldVisitor { ...@@ -113,7 +113,7 @@ class SegmentFieldVisitor {
GetElementVisitor(const FieldElementType elem_type) const { GetElementVisitor(const FieldElementType elem_type) const {
for (auto& kv : elements_map_) { for (auto& kv : elements_map_) {
auto& ev = kv.second; auto& ev = kv.second;
if (ev->GetElement()->GetFtype() == elem_type) { if (ev->GetElement()->GetFEtype() == elem_type) {
return ev; return ev;
} }
} }
......
...@@ -61,7 +61,7 @@ using ResultIds = std::vector<faiss::Index::idx_t>; ...@@ -61,7 +61,7 @@ using ResultIds = std::vector<faiss::Index::idx_t>;
using ResultDistances = std::vector<faiss::Index::distance_t>; using ResultDistances = std::vector<faiss::Index::distance_t>;
/////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////
enum DataType { enum class DataType {
NONE = 0, NONE = 0,
BOOL = 1, BOOL = 1,
INT8 = 2, INT8 = 2,
...@@ -79,7 +79,7 @@ enum DataType { ...@@ -79,7 +79,7 @@ enum DataType {
}; };
/////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////
enum FieldElementType { enum class FieldElementType {
FET_NONE = 0, FET_NONE = 0,
FET_RAW = 1, FET_RAW = 1,
FET_BLOOM_FILTER = 2, FET_BLOOM_FILTER = 2,
......
...@@ -21,6 +21,7 @@ extern const char* F_COLLECTON_ID; ...@@ -21,6 +21,7 @@ extern const char* F_COLLECTON_ID;
extern const char* F_SCHEMA_ID; extern const char* F_SCHEMA_ID;
extern const char* F_NUM; extern const char* F_NUM;
extern const char* F_FTYPE; extern const char* F_FTYPE;
extern const char* F_FETYPE;
extern const char* F_FIELD_ID; extern const char* F_FIELD_ID;
extern const char* F_FIELD_ELEMENT_ID; extern const char* F_FIELD_ELEMENT_ID;
extern const char* F_PARTITION_ID; extern const char* F_PARTITION_ID;
......
...@@ -26,6 +26,7 @@ const char* F_COLLECTON_ID = snapshot::CollectionIdField::Name; ...@@ -26,6 +26,7 @@ const char* F_COLLECTON_ID = snapshot::CollectionIdField::Name;
const char* F_SCHEMA_ID = snapshot::SchemaIdField::Name; const char* F_SCHEMA_ID = snapshot::SchemaIdField::Name;
const char* F_NUM = snapshot::NumField::Name; const char* F_NUM = snapshot::NumField::Name;
const char* F_FTYPE = snapshot::FtypeField::Name; const char* F_FTYPE = snapshot::FtypeField::Name;
const char* F_FETYPE = snapshot::FEtypeField::Name;
const char* F_FIELD_ID = snapshot::FieldIdField::Name; const char* F_FIELD_ID = snapshot::FieldIdField::Name;
const char* F_FIELD_ELEMENT_ID = snapshot::FieldElementIdField::Name; const char* F_FIELD_ELEMENT_ID = snapshot::FieldElementIdField::Name;
const char* F_PARTITION_ID = snapshot::PartitionIdField::Name; const char* F_PARTITION_ID = snapshot::PartitionIdField::Name;
...@@ -66,14 +67,14 @@ ResourceAttrMapOf(const std::string& table, std::vector<std::string>& attrs) { ...@@ -66,14 +67,14 @@ ResourceAttrMapOf(const std::string& table, std::vector<std::string>& attrs) {
{F_SCHEMA_ID, F_PARTITION_ID, F_SEGMENT_ID, F_MAPPINGS, F_ROW_COUNT, F_SIZE, F_ID, F_LSN, F_STATE, {F_SCHEMA_ID, F_PARTITION_ID, F_SEGMENT_ID, F_MAPPINGS, F_ROW_COUNT, F_SIZE, F_ID, F_LSN, F_STATE,
F_CREATED_ON, F_UPDATED_ON}}, F_CREATED_ON, F_UPDATED_ON}},
{snapshot::SegmentFile::Name, {snapshot::SegmentFile::Name,
{F_COLLECTON_ID, F_PARTITION_ID, F_SEGMENT_ID, F_FIELD_ELEMENT_ID, F_FTYPE, F_ROW_COUNT, F_SIZE, F_ID, F_LSN, {F_COLLECTON_ID, F_PARTITION_ID, F_SEGMENT_ID, F_FIELD_ELEMENT_ID, F_FETYPE, F_ROW_COUNT, F_SIZE, F_ID, F_LSN,
F_STATE, F_CREATED_ON, F_UPDATED_ON}}, F_STATE, F_CREATED_ON, F_UPDATED_ON}},
{snapshot::SchemaCommit::Name, {F_COLLECTON_ID, F_MAPPINGS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}}, {snapshot::SchemaCommit::Name, {F_COLLECTON_ID, F_MAPPINGS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}},
{snapshot::Field::Name, {F_NAME, F_NUM, F_FTYPE, F_PARAMS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}}, {snapshot::Field::Name, {F_NAME, F_NUM, F_FTYPE, F_PARAMS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}},
{snapshot::FieldCommit::Name, {snapshot::FieldCommit::Name,
{F_COLLECTON_ID, F_FIELD_ID, F_MAPPINGS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}}, {F_COLLECTON_ID, F_FIELD_ID, F_MAPPINGS, F_ID, F_LSN, F_STATE, F_CREATED_ON, F_UPDATED_ON}},
{snapshot::FieldElement::Name, {snapshot::FieldElement::Name,
{F_COLLECTON_ID, F_FIELD_ID, F_NAME, F_FTYPE, F_TYPE_NAME, F_PARAMS, F_ID, F_LSN, F_STATE, F_CREATED_ON, {F_COLLECTON_ID, F_FIELD_ID, F_NAME, F_FETYPE, F_TYPE_NAME, F_PARAMS, F_ID, F_LSN, F_STATE, F_CREATED_ON,
F_UPDATED_ON}}, F_UPDATED_ON}},
}; };
......
...@@ -110,7 +110,11 @@ AttrValue2Str(typename ResourceContext<ResourceT>::ResPtr src, const std::string ...@@ -110,7 +110,11 @@ AttrValue2Str(typename ResourceContext<ResourceT>::ResPtr src, const std::string
int2str(int_value, value); int2str(int_value, value);
} else if (F_FTYPE == attr) { } else if (F_FTYPE == attr) {
auto ftype_field = std::dynamic_pointer_cast<snapshot::FtypeField>(src); auto ftype_field = std::dynamic_pointer_cast<snapshot::FtypeField>(src);
int_value = ftype_field->GetFtype(); int_value = (int)ftype_field->GetFtype();
int2str(int_value, value);
} else if (F_FETYPE == attr) {
auto fetype_field = std::dynamic_pointer_cast<snapshot::FEtypeField>(src);
int_value = (int)fetype_field->GetFEtype();
int2str(int_value, value); int2str(int_value, value);
} else if (F_FIELD_ID == attr) { } else if (F_FIELD_ID == attr) {
auto field_id_field = std::dynamic_pointer_cast<snapshot::FieldIdField>(src); auto field_id_field = std::dynamic_pointer_cast<snapshot::FieldIdField>(src);
......
...@@ -31,6 +31,8 @@ ...@@ -31,6 +31,8 @@
#include "utils/Status.h" #include "utils/Status.h"
namespace milvus::engine::meta { namespace milvus::engine::meta {
using snapshot::FETYPE_TYPE;
using snapshot::FTYPE_TYPE;
class MetaSession { class MetaSession {
public: public:
...@@ -228,11 +230,20 @@ MetaSession::Select(const std::string& field, const std::vector<U>& values, ...@@ -228,11 +230,20 @@ MetaSession::Select(const std::string& field, const std::vector<U>& values,
if (ftype_p != nullptr) { if (ftype_p != nullptr) {
iter = raw.find(F_FTYPE); iter = raw.find(F_FTYPE);
if (iter != raw.end()) { if (iter != raw.end()) {
auto ftype = std::stol(iter->second); auto ftype = (FTYPE_TYPE)std::stol(iter->second);
ftype_p->SetFtype(ftype); ftype_p->SetFtype(ftype);
} }
} }
auto fetype_p = std::dynamic_pointer_cast<snapshot::FEtypeField>(resource);
if (fetype_p != nullptr) {
iter = raw.find(F_FETYPE);
if (iter != raw.end()) {
auto fetype = (FETYPE_TYPE)std::stol(iter->second);
fetype_p->SetFEtype(fetype);
}
}
auto fid_p = std::dynamic_pointer_cast<snapshot::FieldIdField>(resource); auto fid_p = std::dynamic_pointer_cast<snapshot::FieldIdField>(resource);
if (fid_p != nullptr) { if (fid_p != nullptr) {
iter = raw.find(F_FIELD_ID); iter = raw.find(F_FIELD_ID);
......
...@@ -45,6 +45,7 @@ static const MetaField MetaMappingsField = MetaField(F_MAPPINGS, "JSON", "NOT NU ...@@ -45,6 +45,7 @@ static const MetaField MetaMappingsField = MetaField(F_MAPPINGS, "JSON", "NOT NU
static const MetaField MetaNumField = MetaField(F_NUM, "BIGINT", "NOT NULL"); static const MetaField MetaNumField = MetaField(F_NUM, "BIGINT", "NOT NULL");
static const MetaField MetaLSNField = MetaField(F_LSN, "BIGINT", "NOT NULL"); static const MetaField MetaLSNField = MetaField(F_LSN, "BIGINT", "NOT NULL");
static const MetaField MetaFtypeField = MetaField(F_FTYPE, "BIGINT", "NOT NULL"); static const MetaField MetaFtypeField = MetaField(F_FTYPE, "BIGINT", "NOT NULL");
static const MetaField MetaFEtypeField = MetaField(F_FETYPE, "BIGINT", "NOT NULL");
static const MetaField MetaStateField = MetaField(F_STATE, "TINYINT", "NOT NULL"); static const MetaField MetaStateField = MetaField(F_STATE, "TINYINT", "NOT NULL");
static const MetaField MetaCreatedOnField = MetaField(F_CREATED_ON, "BIGINT", "NOT NULL"); static const MetaField MetaCreatedOnField = MetaField(F_CREATED_ON, "BIGINT", "NOT NULL");
static const MetaField MetaUpdatedOnField = MetaField(F_UPDATED_ON, "BIGINT", "NOT NULL"); static const MetaField MetaUpdatedOnField = MetaField(F_UPDATED_ON, "BIGINT", "NOT NULL");
...@@ -85,7 +86,7 @@ static const MetaSchema SEGMENTCOMMIT_SCHEMA(TABLE_SEGMENT_COMMIT, ...@@ -85,7 +86,7 @@ static const MetaSchema SEGMENTCOMMIT_SCHEMA(TABLE_SEGMENT_COMMIT,
static const MetaSchema SEGMENTFILE_SCHEMA(TABLE_SEGMENT_FILE, static const MetaSchema SEGMENTFILE_SCHEMA(TABLE_SEGMENT_FILE,
{MetaIdField, MetaCollectionIdField, MetaPartitionIdField, {MetaIdField, MetaCollectionIdField, MetaPartitionIdField,
MetaSegmentIdField, MetaFieldElementIdField, MetaFtypeField, MetaSegmentIdField, MetaFieldElementIdField, MetaFEtypeField,
MetaRowCountField, MetaSizeField, MetaLSNField, MetaStateField, MetaRowCountField, MetaSizeField, MetaLSNField, MetaStateField,
MetaCreatedOnField, MetaUpdatedOnField}); MetaCreatedOnField, MetaUpdatedOnField});
...@@ -103,7 +104,7 @@ static const MetaSchema FIELDCOMMIT_SCHEMA(TABLE_FIELD_COMMIT, ...@@ -103,7 +104,7 @@ static const MetaSchema FIELDCOMMIT_SCHEMA(TABLE_FIELD_COMMIT,
static const MetaSchema FIELDELEMENT_SCHEMA(TABLE_FIELD_ELEMENT, static const MetaSchema FIELDELEMENT_SCHEMA(TABLE_FIELD_ELEMENT,
{MetaIdField, MetaCollectionIdField, MetaFieldIdField, MetaNameField, {MetaIdField, MetaCollectionIdField, MetaFieldIdField, MetaNameField,
MetaFtypeField, MetaTypeNameField, MetaParamsField, MetaLSNField, MetaFEtypeField, MetaTypeNameField, MetaParamsField, MetaLSNField,
MetaStateField, MetaCreatedOnField, MetaUpdatedOnField}); MetaStateField, MetaCreatedOnField, MetaUpdatedOnField});
} // namespace } // namespace
......
...@@ -37,6 +37,7 @@ static const MetaField MetaMappingsField = MetaField(F_MAPPINGS, "VARCHAR(255)", ...@@ -37,6 +37,7 @@ static const MetaField MetaMappingsField = MetaField(F_MAPPINGS, "VARCHAR(255)",
static const MetaField MetaNumField = MetaField(F_NUM, "BIGINT", "NOT NULL"); static const MetaField MetaNumField = MetaField(F_NUM, "BIGINT", "NOT NULL");
static const MetaField MetaLSNField = MetaField(F_LSN, "BIGINT", "NOT NULL"); static const MetaField MetaLSNField = MetaField(F_LSN, "BIGINT", "NOT NULL");
static const MetaField MetaFtypeField = MetaField(F_FTYPE, "BIGINT", "NOT NULL"); static const MetaField MetaFtypeField = MetaField(F_FTYPE, "BIGINT", "NOT NULL");
static const MetaField MetaFEtypeField = MetaField(F_FETYPE, "BIGINT", "NOT NULL");
static const MetaField MetaStateField = MetaField(F_STATE, "TINYINT", "NOT NULL"); static const MetaField MetaStateField = MetaField(F_STATE, "TINYINT", "NOT NULL");
static const MetaField MetaCreatedOnField = MetaField(F_CREATED_ON, "BIGINT", "NOT NULL"); static const MetaField MetaCreatedOnField = MetaField(F_CREATED_ON, "BIGINT", "NOT NULL");
static const MetaField MetaUpdatedOnField = MetaField(F_UPDATED_ON, "BIGINT", "NOT NULL"); static const MetaField MetaUpdatedOnField = MetaField(F_UPDATED_ON, "BIGINT", "NOT NULL");
...@@ -93,7 +94,7 @@ static const MetaSchema SEGMENTCOMMIT_SCHEMA(TABLE_SEGMENT_COMMIT, { ...@@ -93,7 +94,7 @@ static const MetaSchema SEGMENTCOMMIT_SCHEMA(TABLE_SEGMENT_COMMIT, {
static const MetaSchema SEGMENTFILE_SCHEMA(TABLE_SEGMENT_FILE, static const MetaSchema SEGMENTFILE_SCHEMA(TABLE_SEGMENT_FILE,
{MetaIdField, MetaCollectionIdField, MetaPartitionIdField, {MetaIdField, MetaCollectionIdField, MetaPartitionIdField,
MetaSegmentIdField, MetaFieldElementIdField, MetaFtypeField, MetaSegmentIdField, MetaFieldElementIdField, MetaFEtypeField,
MetaRowCountField, MetaSizeField, MetaLSNField, MetaStateField, MetaRowCountField, MetaSizeField, MetaLSNField, MetaStateField,
MetaCreatedOnField, MetaUpdatedOnField}); MetaCreatedOnField, MetaUpdatedOnField});
...@@ -117,7 +118,7 @@ static const MetaSchema FIELDCOMMIT_SCHEMA(TABLE_FIELD_COMMIT, ...@@ -117,7 +118,7 @@ static const MetaSchema FIELDCOMMIT_SCHEMA(TABLE_FIELD_COMMIT,
static const MetaSchema FIELDELEMENT_SCHEMA(TABLE_FIELD_ELEMENT, static const MetaSchema FIELDELEMENT_SCHEMA(TABLE_FIELD_ELEMENT,
{MetaIdField, MetaCollectionIdField, MetaFieldIdField, MetaNameField, {MetaIdField, MetaCollectionIdField, MetaFieldIdField, MetaNameField,
MetaFtypeField, MetaTypeNameField, MetaParamsField, MetaLSNField, MetaFEtypeField, MetaTypeNameField, MetaParamsField, MetaLSNField,
MetaStateField, MetaCreatedOnField, MetaUpdatedOnField}); MetaStateField, MetaCreatedOnField, MetaUpdatedOnField});
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
......
...@@ -961,8 +961,8 @@ CreateCollectionOperation::DoExecute(StorePtr store) { ...@@ -961,8 +961,8 @@ CreateCollectionOperation::DoExecute(StorePtr store) {
for (auto& element_schema : field_elements) { for (auto& element_schema : field_elements) {
FieldElementPtr element; FieldElementPtr element;
STATUS_CHECK(store->CreateResource<FieldElement>( STATUS_CHECK(store->CreateResource<FieldElement>(
FieldElement(collection->GetID(), field->GetID(), element_schema->GetName(), element_schema->GetFtype(), FieldElement(collection->GetID(), field->GetID(), element_schema->GetName(),
element_schema->GetTypeName()), element_schema->GetFEtype(), element_schema->GetTypeName()),
element)); element));
auto t_fe_ctx_p = ResourceContextBuilder<FieldElement>().SetOp(meta::oUpdate).CreatePtr(); auto t_fe_ctx_p = ResourceContextBuilder<FieldElement>().SetOp(meta::oUpdate).CreatePtr();
AddStepWithLsn(*element, c_context_.lsn, t_fe_ctx_p); AddStepWithLsn(*element, c_context_.lsn, t_fe_ctx_p);
......
...@@ -123,7 +123,7 @@ CreateResPtr<SegmentCommit>() { ...@@ -123,7 +123,7 @@ CreateResPtr<SegmentCommit>() {
template <> template <>
inline SegmentFile::Ptr inline SegmentFile::Ptr
CreateResPtr<SegmentFile>() { CreateResPtr<SegmentFile>() {
return std::make_shared<SegmentFile>(0, 0, 0, 0, 0); return std::make_shared<SegmentFile>(0, 0, 0, 0, FieldElementType::FET_NONE);
} }
template <> template <>
...@@ -135,7 +135,7 @@ CreateResPtr<SchemaCommit>() { ...@@ -135,7 +135,7 @@ CreateResPtr<SchemaCommit>() {
template <> template <>
inline Field::Ptr inline Field::Ptr
CreateResPtr<Field>() { CreateResPtr<Field>() {
return std::make_shared<Field>("", 0, 0); return std::make_shared<Field>("", 0, DataType::NONE);
} }
template <> template <>
...@@ -147,7 +147,7 @@ CreateResPtr<FieldCommit>() { ...@@ -147,7 +147,7 @@ CreateResPtr<FieldCommit>() {
template <> template <>
inline FieldElement::Ptr inline FieldElement::Ptr
CreateResPtr<FieldElement>() { CreateResPtr<FieldElement>() {
return std::make_shared<FieldElement>(0, 0, "", 0); return std::make_shared<FieldElement>(0, 0, "", FieldElementType::FET_NONE);
} }
} // namespace milvus::engine::snapshot } // namespace milvus::engine::snapshot
...@@ -339,7 +339,7 @@ SegmentFileOperation::DoExecute(StorePtr store) { ...@@ -339,7 +339,7 @@ SegmentFileOperation::DoExecute(StorePtr store) {
FieldElementPtr fe; FieldElementPtr fe;
STATUS_CHECK(GetStartedSS()->GetFieldElement(context_.field_name, context_.field_element_name, fe)); STATUS_CHECK(GetStartedSS()->GetFieldElement(context_.field_name, context_.field_element_name, fe));
resource_ = std::make_shared<SegmentFile>(context_.collection_id, context_.partition_id, context_.segment_id, resource_ = std::make_shared<SegmentFile>(context_.collection_id, context_.partition_id, context_.segment_id,
fe->GetID(), fe->GetFtype()); fe->GetID(), fe->GetFEtype());
// auto seg_ctx_p = ResourceContextBuilder<SegmentFile>().SetResource(resource_).SetOp(oAdd).CreatePtr(); // auto seg_ctx_p = ResourceContextBuilder<SegmentFile>().SetResource(resource_).SetOp(oAdd).CreatePtr();
AddStep(*resource_, nullptr, false); AddStep(*resource_, nullptr, false);
return Status::OK(); return Status::OK();
......
...@@ -23,7 +23,8 @@ namespace snapshot { ...@@ -23,7 +23,8 @@ namespace snapshot {
using ID_TYPE = int64_t; using ID_TYPE = int64_t;
using NUM_TYPE = int64_t; using NUM_TYPE = int64_t;
using FTYPE_TYPE = int64_t; using FTYPE_TYPE = DataType;
using FETYPE_TYPE = FieldElementType;
using TS_TYPE = int64_t; using TS_TYPE = int64_t;
using LSN_TYPE = int64_t; using LSN_TYPE = int64_t;
using SIZE_TYPE = uint64_t; using SIZE_TYPE = uint64_t;
......
...@@ -133,13 +133,13 @@ SegmentCommit::ToString() const { ...@@ -133,13 +133,13 @@ SegmentCommit::ToString() const {
} }
SegmentFile::SegmentFile(ID_TYPE collection_id, ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, SegmentFile::SegmentFile(ID_TYPE collection_id, ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id,
FTYPE_TYPE ftype, SIZE_TYPE row_cnt, SIZE_TYPE size, ID_TYPE id, LSN_TYPE lsn, State state, FETYPE_TYPE fetype, SIZE_TYPE row_cnt, SIZE_TYPE size, ID_TYPE id, LSN_TYPE lsn, State state,
TS_TYPE created_on, TS_TYPE updated_on) TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
PartitionIdField(partition_id), PartitionIdField(partition_id),
SegmentIdField(segment_id), SegmentIdField(segment_id),
FieldElementIdField(field_element_id), FieldElementIdField(field_element_id),
FtypeField(ftype), FEtypeField(fetype),
RowCountField(row_cnt), RowCountField(row_cnt),
SizeField(size), SizeField(size),
IdField(id), IdField(id),
...@@ -185,13 +185,13 @@ FieldCommit::FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT ...@@ -185,13 +185,13 @@ FieldCommit::FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
FieldElement::FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, FieldElement::FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FETYPE_TYPE fetype,
const std::string& type_name, const json& params, ID_TYPE id, LSN_TYPE lsn, State state, const std::string& type_name, const json& params, ID_TYPE id, LSN_TYPE lsn, State state,
TS_TYPE created_on, TS_TYPE updated_on) TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
FieldIdField(field_id), FieldIdField(field_id),
NameField(name), NameField(name),
FtypeField(ftype), FEtypeField(fetype),
TypeNameField(type_name), TypeNameField(type_name),
ParamsField(params), ParamsField(params),
IdField(id), IdField(id),
......
...@@ -248,7 +248,7 @@ class FtypeField { ...@@ -248,7 +248,7 @@ class FtypeField {
public: public:
static constexpr const char* Name = "ftype"; static constexpr const char* Name = "ftype";
explicit FtypeField(FTYPE_TYPE type) : ftype_(type) { explicit FtypeField(FTYPE_TYPE ftype) : ftype_(ftype) {
} }
FTYPE_TYPE FTYPE_TYPE
...@@ -257,14 +257,35 @@ class FtypeField { ...@@ -257,14 +257,35 @@ class FtypeField {
} }
void void
SetFtype(FTYPE_TYPE type) { SetFtype(FTYPE_TYPE ftype) {
ftype_ = type; ftype_ = ftype;
} }
protected: protected:
FTYPE_TYPE ftype_; FTYPE_TYPE ftype_;
}; };
class FEtypeField {
public:
static constexpr const char* Name = "fetype";
explicit FEtypeField(FETYPE_TYPE fetype) : fetype_(fetype) {
}
FETYPE_TYPE
GetFEtype() const {
return fetype_;
}
void
SetFEtype(FETYPE_TYPE fetype) {
fetype_ = fetype;
}
protected:
FETYPE_TYPE fetype_;
};
class FieldIdField { class FieldIdField {
public: public:
static constexpr const char* Name = "field_id"; static constexpr const char* Name = "field_id";
...@@ -623,7 +644,7 @@ class SegmentFile : public BaseResource<SegmentFile>, ...@@ -623,7 +644,7 @@ class SegmentFile : public BaseResource<SegmentFile>,
public PartitionIdField, public PartitionIdField,
public SegmentIdField, public SegmentIdField,
public FieldElementIdField, public FieldElementIdField,
public FtypeField, public FEtypeField,
public RowCountField, public RowCountField,
public SizeField, public SizeField,
public IdField, public IdField,
...@@ -640,7 +661,7 @@ class SegmentFile : public BaseResource<SegmentFile>, ...@@ -640,7 +661,7 @@ class SegmentFile : public BaseResource<SegmentFile>,
static constexpr const char* Name = "SegmentFile"; static constexpr const char* Name = "SegmentFile";
SegmentFile(ID_TYPE collection_id, ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, SegmentFile(ID_TYPE collection_id, ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id,
FTYPE_TYPE ftype, SIZE_TYPE row_cnt = 0, SIZE_TYPE size = 0, ID_TYPE id = 0, LSN_TYPE lsn = 0, FETYPE_TYPE fetype, SIZE_TYPE row_cnt = 0, SIZE_TYPE size = 0, ID_TYPE id = 0, LSN_TYPE lsn = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -729,7 +750,7 @@ class FieldElement : public BaseResource<FieldElement>, ...@@ -729,7 +750,7 @@ class FieldElement : public BaseResource<FieldElement>,
public CollectionIdField, public CollectionIdField,
public FieldIdField, public FieldIdField,
public NameField, public NameField,
public FtypeField, public FEtypeField,
public TypeNameField, public TypeNameField,
public ParamsField, public ParamsField,
public IdField, public IdField,
...@@ -744,7 +765,7 @@ class FieldElement : public BaseResource<FieldElement>, ...@@ -744,7 +765,7 @@ class FieldElement : public BaseResource<FieldElement>,
using ScopedMapT = std::map<ID_TYPE, ScopedResource<FieldElement>>; using ScopedMapT = std::map<ID_TYPE, ScopedResource<FieldElement>>;
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "FieldElement"; static constexpr const char* Name = "FieldElement";
FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FETYPE_TYPE fetype,
const std::string& type_name = "", const json& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, const std::string& type_name = "", const json& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
......
...@@ -247,7 +247,7 @@ Snapshot::ToString() const { ...@@ -247,7 +247,7 @@ Snapshot::ToString() const {
for (auto& fe_id : fc_m) { for (auto& fe_id : fc_m) {
auto fe = GetResource<FieldElement>(fe_id); auto fe = GetResource<FieldElement>(fe_id);
ss << "\n\tFieldElement: id=" << fe_id << ",name=" << fe->GetName() << " CID=" << fe->GetCollectionId(); ss << "\n\tFieldElement: id=" << fe_id << ",name=" << fe->GetName() << " CID=" << fe->GetCollectionId();
ss << ",ftype=" << fe->GetFtype() << ",typename=" << fe->GetTypeName(); ss << ",fetype=" << (int)fe->GetFEtype() << ",typename=" << fe->GetTypeName();
} }
} }
...@@ -273,7 +273,7 @@ Snapshot::ToString() const { ...@@ -273,7 +273,7 @@ Snapshot::ToString() const {
auto sf = GetResource<SegmentFile>(sf_id); auto sf = GetResource<SegmentFile>(sf_id);
ss << "\n\tSegmentFile: id=" << sf_id << ",field_element_id=" << sf->GetFieldElementId(); ss << "\n\tSegmentFile: id=" << sf_id << ",field_element_id=" << sf->GetFieldElementId();
ss << ",size=" << sf->GetSize(); ss << ",size=" << sf->GetSize();
ss << ",ftype=" << sf->GetFtype(); ss << ",fetype=" << (int)sf->GetFEtype();
} }
} }
} }
......
...@@ -296,7 +296,7 @@ class Store : public std::enable_shared_from_this<Store> { ...@@ -296,7 +296,7 @@ class Store : public std::enable_shared_from_this<Store> {
fename << "fe_" << field->GetID() << "_" << ++id_map[FieldElement::Name]; fename << "fe_" << field->GetID() << "_" << ++id_map[FieldElement::Name];
FieldElementPtr element; FieldElementPtr element;
FieldElement temp_fe(c->GetID(), field->GetID(), fename.str(), fei); FieldElement temp_fe(c->GetID(), field->GetID(), fename.str(), (FieldElementType)fei);
temp_fe.Activate(); temp_fe.Activate();
CreateResource<FieldElement>(std::move(temp_fe), element); CreateResource<FieldElement>(std::move(temp_fe), element);
all_records.push_back(element); all_records.push_back(element);
...@@ -338,7 +338,7 @@ class Store : public std::enable_shared_from_this<Store> { ...@@ -338,7 +338,7 @@ class Store : public std::enable_shared_from_this<Store> {
FieldElementPtr fe_p; FieldElementPtr fe_p;
GetResource<FieldElement>(field_element_id, fe_p); GetResource<FieldElement>(field_element_id, fe_p);
CreateResource<SegmentFile>( CreateResource<SegmentFile>(
SegmentFile(c->GetID(), p->GetID(), s->GetID(), field_element_id, fe_p->GetFtype(), 0, SegmentFile(c->GetID(), p->GetID(), s->GetID(), field_element_id, fe_p->GetFEtype(), 0,
0, 0, 0, ACTIVE), 0, 0, 0, ACTIVE),
sf); sf);
all_records.push_back(sf); all_records.push_back(sf);
......
...@@ -33,7 +33,7 @@ WhichFieldsToBuild(const engine::snapshot::ScopedSnapshotT& snapshot, engine::sn ...@@ -33,7 +33,7 @@ WhichFieldsToBuild(const engine::snapshot::ScopedSnapshotT& snapshot, engine::sn
bool is_vector = (ftype == engine::DataType::VECTOR_FLOAT || ftype == engine::DataType::VECTOR_BINARY); bool is_vector = (ftype == engine::DataType::VECTOR_FLOAT || ftype == engine::DataType::VECTOR_BINARY);
auto elements = snapshot->GetFieldElementsByField(field_name); auto elements = snapshot->GetFieldElementsByField(field_name);
for (auto& element : elements) { for (auto& element : elements) {
if (element->GetFtype() != engine::FieldElementType::FET_INDEX) { if (element->GetFEtype() != engine::FieldElementType::FET_INDEX) {
continue; // only check index element continue; // only check index element
} }
......
...@@ -56,7 +56,7 @@ GetCollectionInfoReq::OnExecute() { ...@@ -56,7 +56,7 @@ GetCollectionInfoReq::OnExecute() {
milvus::json field_index_param; milvus::json field_index_param;
auto field_elements = field_kv.second; auto field_elements = field_kv.second;
for (const auto& element : field_elements) { for (const auto& element : field_elements) {
if (element->GetFtype() == (engine::snapshot::FTYPE_TYPE)engine::FieldElementType::FET_INDEX) { if (element->GetFEtype() == engine::FieldElementType::FET_INDEX) {
field_index_param = element->GetParams(); field_index_param = element->GetParams();
auto type = element->GetTypeName(); auto type = element->GetTypeName();
field_schema.index_params_ = field_index_param; field_schema.index_params_ = field_index_param;
...@@ -66,7 +66,7 @@ GetCollectionInfoReq::OnExecute() { ...@@ -66,7 +66,7 @@ GetCollectionInfoReq::OnExecute() {
} }
auto field_name = field->GetName(); auto field_name = field->GetName();
field_schema.field_type_ = (engine::DataType)field->GetFtype(); field_schema.field_type_ = field->GetFtype();
field_schema.field_params_ = field->GetParams(); field_schema.field_params_ = field->GetParams();
collection_schema_.fields_.insert(std::make_pair(field_name, field_schema)); collection_schema_.fields_.insert(std::make_pair(field_name, field_schema));
......
...@@ -75,12 +75,12 @@ SearchReq::OnExecute() { ...@@ -75,12 +75,12 @@ SearchReq::OnExecute() {
std::unordered_map<std::string, engine::DataType> field_types; std::unordered_map<std::string, engine::DataType> field_types;
for (auto& schema : fields_schema) { for (auto& schema : fields_schema) {
auto field = schema.first; auto field = schema.first;
field_types.insert(std::make_pair(field->GetName(), (engine::DataType)field->GetFtype())); field_types.insert(std::make_pair(field->GetName(), field->GetFtype()));
if (field->GetFtype() == (int)engine::DataType::VECTOR_FLOAT || if (field->GetFtype() == engine::DataType::VECTOR_FLOAT ||
field->GetFtype() == (int)engine::DataType::VECTOR_BINARY) { field->GetFtype() == engine::DataType::VECTOR_BINARY) {
dimension = field->GetParams()[engine::PARAM_DIMENSION]; dimension = field->GetParams()[engine::PARAM_DIMENSION];
// validate search metric type and DataType match // validate search metric type and DataType match
bool is_binary = (field->GetFtype() == (int)engine::DataType::VECTOR_FLOAT) ? false : true; bool is_binary = (field->GetFtype() == engine::DataType::VECTOR_FLOAT) ? false : true;
if (query_ptr_->metric_types.find(field->GetName()) != query_ptr_->metric_types.end()) { if (query_ptr_->metric_types.find(field->GetName()) != query_ptr_->metric_types.end()) {
auto metric_type = query_ptr_->metric_types.at(field->GetName()); auto metric_type = query_ptr_->metric_types.at(field->GetName());
STATUS_CHECK(ValidateSearchMetricType(metric_type, is_binary)); STATUS_CHECK(ValidateSearchMetricType(metric_type, is_binary));
......
...@@ -561,7 +561,8 @@ class WebController : public oatpp::web::server::api::ApiController { ...@@ -561,7 +561,8 @@ class WebController : public oatpp::web::server::api::ApiController {
QUERIES(QueryParams, query_params)) { QUERIES(QueryParams, query_params)) {
auto handler = WebRequestHandler(); auto handler = WebRequestHandler();
String response;
String response;
auto status_dto = handler.GetEntity(collection_name, query_params, response); auto status_dto = handler.GetEntity(collection_name, query_params, response);
switch (*(status_dto->code)) { switch (*(status_dto->code)) {
case StatusCode::SUCCESS: case StatusCode::SUCCESS:
......
...@@ -821,7 +821,7 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js ...@@ -821,7 +821,7 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
for (const auto& field : field_mappings) { for (const auto& field : field_mappings) {
auto field_name = field.first->GetName(); auto field_name = field.first->GetName();
auto field_data = result->data_chunk_->fixed_fields_; auto field_data = result->data_chunk_->fixed_fields_;
switch ((int64_t)field.first->GetFtype()) { switch (field.first->GetFtype()) {
case engine::DataType::INT32: { case engine::DataType::INT32: {
int32_t int32_value; int32_t int32_value;
int64_t offset = (i * step + j) * sizeof(int32_t); int64_t offset = (i * step + j) * sizeof(int32_t);
......
...@@ -87,8 +87,7 @@ TEST_F(EventTest, TestInActiveResGcEvent) { ...@@ -87,8 +87,7 @@ TEST_F(EventTest, TestInActiveResGcEvent) {
SegmentFilePtr seg_file; SegmentFilePtr seg_file;
status = store_->CreateResource<SegmentFile>( status = store_->CreateResource<SegmentFile>(
SegmentFile(collection->GetID(), partition->GetID(), s->GetID(), field_element->GetID(), SegmentFile(collection->GetID(), partition->GetID(), s->GetID(), field_element->GetID(), field_element->GetFEtype()), seg_file);
field_element->GetFtype()), seg_file);
ASSERT_TRUE(status.ok()) << status.ToString(); ASSERT_TRUE(status.ok()) << status.ToString();
SegmentCommitPtr sc; SegmentCommitPtr sc;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册