diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h index 56fa68890795927ccb96a9093a704f9d8115a02d..0ee2d5b3a01cd3f1a1579b405ff4515e5df97624 100644 --- a/internal/core/src/common/Types.h +++ b/internal/core/src/common/Types.h @@ -18,11 +18,14 @@ #include #include #include +#include #include #include "pb/schema.pb.h" namespace milvus { using Timestamp = uint64_t; // TODO: use TiKV-like timestamp +constexpr auto MAX_TIMESTAMP = std::numeric_limits::max(); + using engine::DataType; using engine::FieldElementType; using engine::idx_t; diff --git a/internal/core/src/pb/segcore.pb.cc b/internal/core/src/pb/segcore.pb.cc index bf73c04e666513217b912e8618695c5eb4971d0a..257a0c3bb2aa34b74e0bc62e63a6a7abdfff8956 100644 --- a/internal/core/src/pb/segcore.pb.cc +++ b/internal/core/src/pb/segcore.pb.cc @@ -17,6 +17,7 @@ #include extern PROTOBUF_INTERNAL_EXPORT_schema_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_FieldData_schema_2eproto; extern PROTOBUF_INTERNAL_EXPORT_schema_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_IDs_schema_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_segcore_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_LoadFieldMeta_segcore_2eproto; namespace milvus { namespace proto { namespace segcore { @@ -28,9 +29,46 @@ class RetrieveResultsDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _RetrieveResults_default_instance_; +class LoadFieldMetaDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _LoadFieldMeta_default_instance_; +class LoadSegmentMetaDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _LoadSegmentMeta_default_instance_; } // namespace segcore } // namespace proto } // namespace milvus +static void InitDefaultsscc_info_LoadFieldMeta_segcore_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::proto::segcore::_LoadFieldMeta_default_instance_; + new (ptr) ::milvus::proto::segcore::LoadFieldMeta(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::proto::segcore::LoadFieldMeta::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_LoadFieldMeta_segcore_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_LoadFieldMeta_segcore_2eproto}, {}}; + +static void InitDefaultsscc_info_LoadSegmentMeta_segcore_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::proto::segcore::_LoadSegmentMeta_default_instance_; + new (ptr) ::milvus::proto::segcore::LoadSegmentMeta(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::proto::segcore::LoadSegmentMeta::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_LoadSegmentMeta_segcore_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_LoadSegmentMeta_segcore_2eproto}, { + &scc_info_LoadFieldMeta_segcore_2eproto.base,}}; + static void InitDefaultsscc_info_RetrieveRequest_segcore_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -62,7 +100,7 @@ static void InitDefaultsscc_info_RetrieveResults_segcore_2eproto() { &scc_info_IDs_schema_2eproto.base, &scc_info_FieldData_schema_2eproto.base,}}; -static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_segcore_2eproto[2]; +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_segcore_2eproto[4]; static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_segcore_2eproto = nullptr; static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_segcore_2eproto = nullptr; @@ -81,15 +119,34 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_segcore_2eproto::offsets[] PRO ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::RetrieveResults, ids_), PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::RetrieveResults, fields_data_), + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadFieldMeta, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadFieldMeta, min_timestamp_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadFieldMeta, max_timestamp_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadFieldMeta, row_count_), + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadSegmentMeta, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadSegmentMeta, metas_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::segcore::LoadSegmentMeta, total_size_), }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::proto::segcore::RetrieveRequest)}, { 7, -1, sizeof(::milvus::proto::segcore::RetrieveResults)}, + { 14, -1, sizeof(::milvus::proto::segcore::LoadFieldMeta)}, + { 22, -1, sizeof(::milvus::proto::segcore::LoadSegmentMeta)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { reinterpret_cast(&::milvus::proto::segcore::_RetrieveRequest_default_instance_), reinterpret_cast(&::milvus::proto::segcore::_RetrieveResults_default_instance_), + reinterpret_cast(&::milvus::proto::segcore::_LoadFieldMeta_default_instance_), + reinterpret_cast(&::milvus::proto::segcore::_LoadSegmentMeta_default_instance_), }; const char descriptor_table_protodef_segcore_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = @@ -99,23 +156,29 @@ const char descriptor_table_protodef_segcore_2eproto[] PROTOBUF_SECTION_VARIABLE "fields\030\002 \003(\t\"m\n\017RetrieveResults\022%\n\003ids\030\001" " \001(\0132\030.milvus.proto.schema.IDs\0223\n\013fields" "_data\030\002 \003(\0132\036.milvus.proto.schema.FieldD" - "ataB6Z4github.com/milvus-io/milvus/inter" - "nal/proto/segcorepbb\006proto3" + "ata\"P\n\rLoadFieldMeta\022\025\n\rmin_timestamp\030\001 " + "\001(\003\022\025\n\rmax_timestamp\030\002 \001(\003\022\021\n\trow_count\030" + "\003 \001(\003\"Y\n\017LoadSegmentMeta\0222\n\005metas\030\001 \003(\0132" + "#.milvus.proto.segcore.LoadFieldMeta\022\022\n\n" + "total_size\030\002 \001(\003B6Z4github.com/milvus-io" + "/milvus/internal/proto/segcorepbb\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_segcore_2eproto_deps[1] = { &::descriptor_table_schema_2eproto, }; -static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_segcore_2eproto_sccs[2] = { +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_segcore_2eproto_sccs[4] = { + &scc_info_LoadFieldMeta_segcore_2eproto.base, + &scc_info_LoadSegmentMeta_segcore_2eproto.base, &scc_info_RetrieveRequest_segcore_2eproto.base, &scc_info_RetrieveResults_segcore_2eproto.base, }; static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_segcore_2eproto_once; static bool descriptor_table_segcore_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_segcore_2eproto = { - &descriptor_table_segcore_2eproto_initialized, descriptor_table_protodef_segcore_2eproto, "segcore.proto", 307, - &descriptor_table_segcore_2eproto_once, descriptor_table_segcore_2eproto_sccs, descriptor_table_segcore_2eproto_deps, 2, 1, + &descriptor_table_segcore_2eproto_initialized, descriptor_table_protodef_segcore_2eproto, "segcore.proto", 480, + &descriptor_table_segcore_2eproto_once, descriptor_table_segcore_2eproto_sccs, descriptor_table_segcore_2eproto_deps, 4, 1, schemas, file_default_instances, TableStruct_segcore_2eproto::offsets, - file_level_metadata_segcore_2eproto, 2, file_level_enum_descriptors_segcore_2eproto, file_level_service_descriptors_segcore_2eproto, + file_level_metadata_segcore_2eproto, 4, file_level_enum_descriptors_segcore_2eproto, file_level_service_descriptors_segcore_2eproto, }; // Force running AddDescriptors() at dynamic initialization time. @@ -786,6 +849,648 @@ void RetrieveResults::InternalSwap(RetrieveResults* other) { } +// =================================================================== + +void LoadFieldMeta::InitAsDefaultInstance() { +} +class LoadFieldMeta::_Internal { + public: +}; + +LoadFieldMeta::LoadFieldMeta() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.proto.segcore.LoadFieldMeta) +} +LoadFieldMeta::LoadFieldMeta(const LoadFieldMeta& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::memcpy(&min_timestamp_, &from.min_timestamp_, + static_cast(reinterpret_cast(&row_count_) - + reinterpret_cast(&min_timestamp_)) + sizeof(row_count_)); + // @@protoc_insertion_point(copy_constructor:milvus.proto.segcore.LoadFieldMeta) +} + +void LoadFieldMeta::SharedCtor() { + ::memset(&min_timestamp_, 0, static_cast( + reinterpret_cast(&row_count_) - + reinterpret_cast(&min_timestamp_)) + sizeof(row_count_)); +} + +LoadFieldMeta::~LoadFieldMeta() { + // @@protoc_insertion_point(destructor:milvus.proto.segcore.LoadFieldMeta) + SharedDtor(); +} + +void LoadFieldMeta::SharedDtor() { +} + +void LoadFieldMeta::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const LoadFieldMeta& LoadFieldMeta::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_LoadFieldMeta_segcore_2eproto.base); + return *internal_default_instance(); +} + + +void LoadFieldMeta::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.proto.segcore.LoadFieldMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + ::memset(&min_timestamp_, 0, static_cast( + reinterpret_cast(&row_count_) - + reinterpret_cast(&min_timestamp_)) + sizeof(row_count_)); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* LoadFieldMeta::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // int64 min_timestamp = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { + min_timestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int64 max_timestamp = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + max_timestamp_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int64 row_count = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + row_count_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool LoadFieldMeta::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.proto.segcore.LoadFieldMeta) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // int64 min_timestamp = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &min_timestamp_))); + } else { + goto handle_unusual; + } + break; + } + + // int64 max_timestamp = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &max_timestamp_))); + } else { + goto handle_unusual; + } + break; + } + + // int64 row_count = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &row_count_))); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.proto.segcore.LoadFieldMeta) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.proto.segcore.LoadFieldMeta) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void LoadFieldMeta::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.proto.segcore.LoadFieldMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // int64 min_timestamp = 1; + if (this->min_timestamp() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(1, this->min_timestamp(), output); + } + + // int64 max_timestamp = 2; + if (this->max_timestamp() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->max_timestamp(), output); + } + + // int64 row_count = 3; + if (this->row_count() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->row_count(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.proto.segcore.LoadFieldMeta) +} + +::PROTOBUF_NAMESPACE_ID::uint8* LoadFieldMeta::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.proto.segcore.LoadFieldMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // int64 min_timestamp = 1; + if (this->min_timestamp() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(1, this->min_timestamp(), target); + } + + // int64 max_timestamp = 2; + if (this->max_timestamp() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->max_timestamp(), target); + } + + // int64 row_count = 3; + if (this->row_count() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->row_count(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.proto.segcore.LoadFieldMeta) + return target; +} + +size_t LoadFieldMeta::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.proto.segcore.LoadFieldMeta) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // int64 min_timestamp = 1; + if (this->min_timestamp() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->min_timestamp()); + } + + // int64 max_timestamp = 2; + if (this->max_timestamp() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->max_timestamp()); + } + + // int64 row_count = 3; + if (this->row_count() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->row_count()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void LoadFieldMeta::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.proto.segcore.LoadFieldMeta) + GOOGLE_DCHECK_NE(&from, this); + const LoadFieldMeta* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.proto.segcore.LoadFieldMeta) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.proto.segcore.LoadFieldMeta) + MergeFrom(*source); + } +} + +void LoadFieldMeta::MergeFrom(const LoadFieldMeta& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.segcore.LoadFieldMeta) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.min_timestamp() != 0) { + set_min_timestamp(from.min_timestamp()); + } + if (from.max_timestamp() != 0) { + set_max_timestamp(from.max_timestamp()); + } + if (from.row_count() != 0) { + set_row_count(from.row_count()); + } +} + +void LoadFieldMeta::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.proto.segcore.LoadFieldMeta) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void LoadFieldMeta::CopyFrom(const LoadFieldMeta& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.segcore.LoadFieldMeta) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool LoadFieldMeta::IsInitialized() const { + return true; +} + +void LoadFieldMeta::InternalSwap(LoadFieldMeta* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(min_timestamp_, other->min_timestamp_); + swap(max_timestamp_, other->max_timestamp_); + swap(row_count_, other->row_count_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata LoadFieldMeta::GetMetadata() const { + return GetMetadataStatic(); +} + + +// =================================================================== + +void LoadSegmentMeta::InitAsDefaultInstance() { +} +class LoadSegmentMeta::_Internal { + public: +}; + +LoadSegmentMeta::LoadSegmentMeta() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.proto.segcore.LoadSegmentMeta) +} +LoadSegmentMeta::LoadSegmentMeta(const LoadSegmentMeta& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr), + metas_(from.metas_) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + total_size_ = from.total_size_; + // @@protoc_insertion_point(copy_constructor:milvus.proto.segcore.LoadSegmentMeta) +} + +void LoadSegmentMeta::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_LoadSegmentMeta_segcore_2eproto.base); + total_size_ = PROTOBUF_LONGLONG(0); +} + +LoadSegmentMeta::~LoadSegmentMeta() { + // @@protoc_insertion_point(destructor:milvus.proto.segcore.LoadSegmentMeta) + SharedDtor(); +} + +void LoadSegmentMeta::SharedDtor() { +} + +void LoadSegmentMeta::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const LoadSegmentMeta& LoadSegmentMeta::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_LoadSegmentMeta_segcore_2eproto.base); + return *internal_default_instance(); +} + + +void LoadSegmentMeta::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.proto.segcore.LoadSegmentMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + metas_.Clear(); + total_size_ = PROTOBUF_LONGLONG(0); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* LoadSegmentMeta::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_metas(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 10); + } else goto handle_unusual; + continue; + // int64 total_size = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + total_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool LoadSegmentMeta::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.proto.segcore.LoadSegmentMeta) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_metas())); + } else { + goto handle_unusual; + } + break; + } + + // int64 total_size = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( + input, &total_size_))); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.proto.segcore.LoadSegmentMeta) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.proto.segcore.LoadSegmentMeta) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void LoadSegmentMeta::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.proto.segcore.LoadSegmentMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + for (unsigned int i = 0, + n = static_cast(this->metas_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 1, + this->metas(static_cast(i)), + output); + } + + // int64 total_size = 2; + if (this->total_size() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->total_size(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.proto.segcore.LoadSegmentMeta) +} + +::PROTOBUF_NAMESPACE_ID::uint8* LoadSegmentMeta::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.proto.segcore.LoadSegmentMeta) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + for (unsigned int i = 0, + n = static_cast(this->metas_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 1, this->metas(static_cast(i)), target); + } + + // int64 total_size = 2; + if (this->total_size() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->total_size(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.proto.segcore.LoadSegmentMeta) + return target; +} + +size_t LoadSegmentMeta::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.proto.segcore.LoadSegmentMeta) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + { + unsigned int count = static_cast(this->metas_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->metas(static_cast(i))); + } + } + + // int64 total_size = 2; + if (this->total_size() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( + this->total_size()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void LoadSegmentMeta::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.proto.segcore.LoadSegmentMeta) + GOOGLE_DCHECK_NE(&from, this); + const LoadSegmentMeta* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.proto.segcore.LoadSegmentMeta) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.proto.segcore.LoadSegmentMeta) + MergeFrom(*source); + } +} + +void LoadSegmentMeta::MergeFrom(const LoadSegmentMeta& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.segcore.LoadSegmentMeta) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + metas_.MergeFrom(from.metas_); + if (from.total_size() != 0) { + set_total_size(from.total_size()); + } +} + +void LoadSegmentMeta::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.proto.segcore.LoadSegmentMeta) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void LoadSegmentMeta::CopyFrom(const LoadSegmentMeta& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.segcore.LoadSegmentMeta) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool LoadSegmentMeta::IsInitialized() const { + return true; +} + +void LoadSegmentMeta::InternalSwap(LoadSegmentMeta* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&metas_)->InternalSwap(CastToBase(&other->metas_)); + swap(total_size_, other->total_size_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata LoadSegmentMeta::GetMetadata() const { + return GetMetadataStatic(); +} + + // @@protoc_insertion_point(namespace_scope) } // namespace segcore } // namespace proto @@ -797,6 +1502,12 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::segcore::RetrieveRequest* Arena::C template<> PROTOBUF_NOINLINE ::milvus::proto::segcore::RetrieveResults* Arena::CreateMaybeMessage< ::milvus::proto::segcore::RetrieveResults >(Arena* arena) { return Arena::CreateInternal< ::milvus::proto::segcore::RetrieveResults >(arena); } +template<> PROTOBUF_NOINLINE ::milvus::proto::segcore::LoadFieldMeta* Arena::CreateMaybeMessage< ::milvus::proto::segcore::LoadFieldMeta >(Arena* arena) { + return Arena::CreateInternal< ::milvus::proto::segcore::LoadFieldMeta >(arena); +} +template<> PROTOBUF_NOINLINE ::milvus::proto::segcore::LoadSegmentMeta* Arena::CreateMaybeMessage< ::milvus::proto::segcore::LoadSegmentMeta >(Arena* arena) { + return Arena::CreateInternal< ::milvus::proto::segcore::LoadSegmentMeta >(arena); +} PROTOBUF_NAMESPACE_CLOSE // @@protoc_insertion_point(global_scope) diff --git a/internal/core/src/pb/segcore.pb.h b/internal/core/src/pb/segcore.pb.h index e1a419f11e5369011e41547394f7a3b6e4aede52..56b96778c36215f5ac5f22ed3f6549c387341706 100644 --- a/internal/core/src/pb/segcore.pb.h +++ b/internal/core/src/pb/segcore.pb.h @@ -48,7 +48,7 @@ struct TableStruct_segcore_2eproto { PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] PROTOBUF_SECTION_VARIABLE(protodesc_cold); - static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[2] + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[4] PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; @@ -58,6 +58,12 @@ extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table namespace milvus { namespace proto { namespace segcore { +class LoadFieldMeta; +class LoadFieldMetaDefaultTypeInternal; +extern LoadFieldMetaDefaultTypeInternal _LoadFieldMeta_default_instance_; +class LoadSegmentMeta; +class LoadSegmentMetaDefaultTypeInternal; +extern LoadSegmentMetaDefaultTypeInternal _LoadSegmentMeta_default_instance_; class RetrieveRequest; class RetrieveRequestDefaultTypeInternal; extern RetrieveRequestDefaultTypeInternal _RetrieveRequest_default_instance_; @@ -68,6 +74,8 @@ extern RetrieveResultsDefaultTypeInternal _RetrieveResults_default_instance_; } // namespace proto } // namespace milvus PROTOBUF_NAMESPACE_OPEN +template<> ::milvus::proto::segcore::LoadFieldMeta* Arena::CreateMaybeMessage<::milvus::proto::segcore::LoadFieldMeta>(Arena*); +template<> ::milvus::proto::segcore::LoadSegmentMeta* Arena::CreateMaybeMessage<::milvus::proto::segcore::LoadSegmentMeta>(Arena*); template<> ::milvus::proto::segcore::RetrieveRequest* Arena::CreateMaybeMessage<::milvus::proto::segcore::RetrieveRequest>(Arena*); template<> ::milvus::proto::segcore::RetrieveResults* Arena::CreateMaybeMessage<::milvus::proto::segcore::RetrieveResults>(Arena*); PROTOBUF_NAMESPACE_CLOSE @@ -375,6 +383,295 @@ class RetrieveResults : mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_segcore_2eproto; }; +// ------------------------------------------------------------------- + +class LoadFieldMeta : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.segcore.LoadFieldMeta) */ { + public: + LoadFieldMeta(); + virtual ~LoadFieldMeta(); + + LoadFieldMeta(const LoadFieldMeta& from); + LoadFieldMeta(LoadFieldMeta&& from) noexcept + : LoadFieldMeta() { + *this = ::std::move(from); + } + + inline LoadFieldMeta& operator=(const LoadFieldMeta& from) { + CopyFrom(from); + return *this; + } + inline LoadFieldMeta& operator=(LoadFieldMeta&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const LoadFieldMeta& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const LoadFieldMeta* internal_default_instance() { + return reinterpret_cast( + &_LoadFieldMeta_default_instance_); + } + static constexpr int kIndexInFileMessages = + 2; + + friend void swap(LoadFieldMeta& a, LoadFieldMeta& b) { + a.Swap(&b); + } + inline void Swap(LoadFieldMeta* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline LoadFieldMeta* New() const final { + return CreateMaybeMessage(nullptr); + } + + LoadFieldMeta* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const LoadFieldMeta& from); + void MergeFrom(const LoadFieldMeta& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(LoadFieldMeta* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.proto.segcore.LoadFieldMeta"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_segcore_2eproto); + return ::descriptor_table_segcore_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kMinTimestampFieldNumber = 1, + kMaxTimestampFieldNumber = 2, + kRowCountFieldNumber = 3, + }; + // int64 min_timestamp = 1; + void clear_min_timestamp(); + ::PROTOBUF_NAMESPACE_ID::int64 min_timestamp() const; + void set_min_timestamp(::PROTOBUF_NAMESPACE_ID::int64 value); + + // int64 max_timestamp = 2; + void clear_max_timestamp(); + ::PROTOBUF_NAMESPACE_ID::int64 max_timestamp() const; + void set_max_timestamp(::PROTOBUF_NAMESPACE_ID::int64 value); + + // int64 row_count = 3; + void clear_row_count(); + ::PROTOBUF_NAMESPACE_ID::int64 row_count() const; + void set_row_count(::PROTOBUF_NAMESPACE_ID::int64 value); + + // @@protoc_insertion_point(class_scope:milvus.proto.segcore.LoadFieldMeta) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::int64 min_timestamp_; + ::PROTOBUF_NAMESPACE_ID::int64 max_timestamp_; + ::PROTOBUF_NAMESPACE_ID::int64 row_count_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_segcore_2eproto; +}; +// ------------------------------------------------------------------- + +class LoadSegmentMeta : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.segcore.LoadSegmentMeta) */ { + public: + LoadSegmentMeta(); + virtual ~LoadSegmentMeta(); + + LoadSegmentMeta(const LoadSegmentMeta& from); + LoadSegmentMeta(LoadSegmentMeta&& from) noexcept + : LoadSegmentMeta() { + *this = ::std::move(from); + } + + inline LoadSegmentMeta& operator=(const LoadSegmentMeta& from) { + CopyFrom(from); + return *this; + } + inline LoadSegmentMeta& operator=(LoadSegmentMeta&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const LoadSegmentMeta& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const LoadSegmentMeta* internal_default_instance() { + return reinterpret_cast( + &_LoadSegmentMeta_default_instance_); + } + static constexpr int kIndexInFileMessages = + 3; + + friend void swap(LoadSegmentMeta& a, LoadSegmentMeta& b) { + a.Swap(&b); + } + inline void Swap(LoadSegmentMeta* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline LoadSegmentMeta* New() const final { + return CreateMaybeMessage(nullptr); + } + + LoadSegmentMeta* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const LoadSegmentMeta& from); + void MergeFrom(const LoadSegmentMeta& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(LoadSegmentMeta* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.proto.segcore.LoadSegmentMeta"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_segcore_2eproto); + return ::descriptor_table_segcore_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kMetasFieldNumber = 1, + kTotalSizeFieldNumber = 2, + }; + // repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; + int metas_size() const; + void clear_metas(); + ::milvus::proto::segcore::LoadFieldMeta* mutable_metas(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::segcore::LoadFieldMeta >* + mutable_metas(); + const ::milvus::proto::segcore::LoadFieldMeta& metas(int index) const; + ::milvus::proto::segcore::LoadFieldMeta* add_metas(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::segcore::LoadFieldMeta >& + metas() const; + + // int64 total_size = 2; + void clear_total_size(); + ::PROTOBUF_NAMESPACE_ID::int64 total_size() const; + void set_total_size(::PROTOBUF_NAMESPACE_ID::int64 value); + + // @@protoc_insertion_point(class_scope:milvus.proto.segcore.LoadSegmentMeta) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::segcore::LoadFieldMeta > metas_; + ::PROTOBUF_NAMESPACE_ID::int64 total_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_segcore_2eproto; +}; // =================================================================== @@ -572,11 +869,109 @@ RetrieveResults::fields_data() const { return fields_data_; } +// ------------------------------------------------------------------- + +// LoadFieldMeta + +// int64 min_timestamp = 1; +inline void LoadFieldMeta::clear_min_timestamp() { + min_timestamp_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 LoadFieldMeta::min_timestamp() const { + // @@protoc_insertion_point(field_get:milvus.proto.segcore.LoadFieldMeta.min_timestamp) + return min_timestamp_; +} +inline void LoadFieldMeta::set_min_timestamp(::PROTOBUF_NAMESPACE_ID::int64 value) { + + min_timestamp_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.segcore.LoadFieldMeta.min_timestamp) +} + +// int64 max_timestamp = 2; +inline void LoadFieldMeta::clear_max_timestamp() { + max_timestamp_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 LoadFieldMeta::max_timestamp() const { + // @@protoc_insertion_point(field_get:milvus.proto.segcore.LoadFieldMeta.max_timestamp) + return max_timestamp_; +} +inline void LoadFieldMeta::set_max_timestamp(::PROTOBUF_NAMESPACE_ID::int64 value) { + + max_timestamp_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.segcore.LoadFieldMeta.max_timestamp) +} + +// int64 row_count = 3; +inline void LoadFieldMeta::clear_row_count() { + row_count_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 LoadFieldMeta::row_count() const { + // @@protoc_insertion_point(field_get:milvus.proto.segcore.LoadFieldMeta.row_count) + return row_count_; +} +inline void LoadFieldMeta::set_row_count(::PROTOBUF_NAMESPACE_ID::int64 value) { + + row_count_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.segcore.LoadFieldMeta.row_count) +} + +// ------------------------------------------------------------------- + +// LoadSegmentMeta + +// repeated .milvus.proto.segcore.LoadFieldMeta metas = 1; +inline int LoadSegmentMeta::metas_size() const { + return metas_.size(); +} +inline void LoadSegmentMeta::clear_metas() { + metas_.Clear(); +} +inline ::milvus::proto::segcore::LoadFieldMeta* LoadSegmentMeta::mutable_metas(int index) { + // @@protoc_insertion_point(field_mutable:milvus.proto.segcore.LoadSegmentMeta.metas) + return metas_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::segcore::LoadFieldMeta >* +LoadSegmentMeta::mutable_metas() { + // @@protoc_insertion_point(field_mutable_list:milvus.proto.segcore.LoadSegmentMeta.metas) + return &metas_; +} +inline const ::milvus::proto::segcore::LoadFieldMeta& LoadSegmentMeta::metas(int index) const { + // @@protoc_insertion_point(field_get:milvus.proto.segcore.LoadSegmentMeta.metas) + return metas_.Get(index); +} +inline ::milvus::proto::segcore::LoadFieldMeta* LoadSegmentMeta::add_metas() { + // @@protoc_insertion_point(field_add:milvus.proto.segcore.LoadSegmentMeta.metas) + return metas_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::segcore::LoadFieldMeta >& +LoadSegmentMeta::metas() const { + // @@protoc_insertion_point(field_list:milvus.proto.segcore.LoadSegmentMeta.metas) + return metas_; +} + +// int64 total_size = 2; +inline void LoadSegmentMeta::clear_total_size() { + total_size_ = PROTOBUF_LONGLONG(0); +} +inline ::PROTOBUF_NAMESPACE_ID::int64 LoadSegmentMeta::total_size() const { + // @@protoc_insertion_point(field_get:milvus.proto.segcore.LoadSegmentMeta.total_size) + return total_size_; +} +inline void LoadSegmentMeta::set_total_size(::PROTOBUF_NAMESPACE_ID::int64 value) { + + total_size_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.segcore.LoadSegmentMeta.total_size) +} + #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) diff --git a/internal/core/src/query/generated/ExecExprVisitor.h b/internal/core/src/query/generated/ExecExprVisitor.h index bf432ded44461333969db72ee3603629671acbd9..8f4b38f67f586e7f0e694d2083dab4f65a8aca13 100644 --- a/internal/core/src/query/generated/ExecExprVisitor.h +++ b/internal/core/src/query/generated/ExecExprVisitor.h @@ -37,8 +37,8 @@ class ExecExprVisitor : public ExprVisitor { public: using RetType = std::deque>; - ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count) - : segment_(segment), row_count_(row_count) { + ExecExprVisitor(const segcore::SegmentInternalInterface& segment, int64_t row_count, Timestamp timestamp) + : segment_(segment), row_count_(row_count), timestamp_(timestamp) { } RetType call_child(Expr& expr) { @@ -67,5 +67,6 @@ class ExecExprVisitor : public ExprVisitor { const segcore::SegmentInternalInterface& segment_; int64_t row_count_; std::optional ret_; + Timestamp timestamp_; }; } // namespace milvus::query diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp index fe42de80b41af6aad76f126b899e4cde3fcdfc60..ca9cbdfd5a7230ccfbda3e80c770a1f594a05f56 100644 --- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp @@ -61,6 +61,17 @@ class ExecPlanNodeVisitor : PlanNodeVisitor { } // namespace impl #endif +static QueryResult +empty_query_result(int64_t num_queries, int64_t topk, MetricType metric_type) { + QueryResult final_result; + SubQueryResult result(num_queries, topk, metric_type); + final_result.num_queries_ = num_queries; + final_result.topK_ = topk; + final_result.internal_seg_offsets_ = std::move(result.mutable_labels()); + final_result.result_distances_ = std::move(result.mutable_values()); + return final_result; +} + template void ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) { @@ -76,15 +87,24 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) { aligned_vector bitset_holder; BitsetView view; // TODO: add API to unify row_count - auto row_count = segment->get_row_count(); + // auto row_count = segment->get_row_count(); + auto active_count = segment->get_active_count(timestamp_); + + // skip all calculation + if (active_count == 0) { + ret_ = empty_query_result(num_queries, node.query_info_.topK_, node.query_info_.metric_type_); + return; + } if (node.predicate_.has_value()) { - ExecExprVisitor::RetType expr_ret = ExecExprVisitor(*segment, row_count).call_child(*node.predicate_.value()); + ExecExprVisitor::RetType expr_ret = + ExecExprVisitor(*segment, active_count, timestamp_).call_child(*node.predicate_.value()); + segment->mask_with_timestamps(expr_ret, timestamp_); bitset_holder = AssembleNegBitset(expr_ret); view = BitsetView(bitset_holder.data(), bitset_holder.size() * 8); } - segment->vector_search(row_count, node.query_info_, src_data, num_queries, view, ret); + segment->vector_search(active_count, node.query_info_, src_data, num_queries, MAX_TIMESTAMP, view, ret); ret_ = ret; } diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 3c4b6327eecf180b028bb3ec12e95ad1610358a1..61482d65faed7188e182bd559d97a939167665fc 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -28,6 +28,7 @@ set(SEGCORE_FILES SegcoreConfig.cpp segcore_init_c.cpp ScalarIndex.cpp + TimestampIndex.cpp ) add_library(milvus_segcore SHARED ${SEGCORE_FILES} diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index f24fffce9ef4edde7d20d2bcd8834cc2702b6c49..32aad519ac98942ffaaea528985feef81b34da6a 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -27,6 +27,7 @@ #include "query/PlanImpl.h" #include "segcore/Reduce.h" #include "utils/tools.h" +#include namespace milvus::segcore { @@ -289,6 +290,7 @@ SegmentGrowingImpl::vector_search(int64_t vec_count, query::QueryInfo query_info, const void* query_data, int64_t query_count, + Timestamp timestamp, const BitsetView& bitset, QueryResult& output) const { auto& sealed_indexing = this->get_sealed_indexing_record(); @@ -492,4 +494,19 @@ SegmentGrowingImpl::debug() const { return "Growing\n"; } +int64_t +SegmentGrowingImpl::get_active_count(Timestamp ts) const { + auto row_count = this->get_row_count(); + auto& ts_vec = this->get_insert_record().timestamps_; + auto iter = std::upper_bound(boost::make_counting_iterator((int64_t)0), boost::make_counting_iterator(row_count), + ts, [&](Timestamp ts, int64_t index) { return ts < ts_vec[index]; }); + return *iter; +} + +void +SegmentGrowingImpl::mask_with_timestamps(std::deque>& bitset_chunks, + Timestamp timestamp) const { + // DO NOTHING +} + } // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index 55330d010624b2bc2fbe7485897d416089fba91a..65e83207d44e6f2f60d68507d91fdc7790c4532b 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -33,6 +33,7 @@ #include #include #include +#include namespace milvus::segcore { @@ -136,6 +137,9 @@ class SegmentGrowingImpl : public SegmentGrowing { return 0; } + int64_t + get_active_count(Timestamp ts) const override; + // for scalar vectors template void @@ -170,11 +174,15 @@ class SegmentGrowingImpl : public SegmentGrowing { indexing_record_(*schema_, segcore_config_) { } + void + mask_with_timestamps(std::deque>& bitset_chunks, Timestamp timestamp) const override; + void vector_search(int64_t vec_count, query::QueryInfo query_info, const void* query_data, int64_t query_count, + Timestamp timestamp, const BitsetView& bitset, QueryResult& output) const override; diff --git a/internal/core/src/segcore/SegmentInterface.h b/internal/core/src/segcore/SegmentInterface.h index f9060de9fa2aa632f0d19188a6f3ec2333ad8e1f..f53592b517335e300d780993e7583aa06d8447bb 100644 --- a/internal/core/src/segcore/SegmentInterface.h +++ b/internal/core/src/segcore/SegmentInterface.h @@ -21,6 +21,7 @@ #include "pb/schema.pb.h" #include "pb/segcore.pb.h" #include +#include #include #include #include @@ -104,6 +105,7 @@ class SegmentInternalInterface : public SegmentInterface { query::QueryInfo query_info, const void* query_data, int64_t query_count, + Timestamp timestamp, const BitsetView& bitset, QueryResult& output) const = 0; @@ -111,6 +113,9 @@ class SegmentInternalInterface : public SegmentInterface { virtual int64_t num_chunk_index(FieldOffset field_offset) const = 0; + virtual void + mask_with_timestamps(std::deque>& bitset_chunks, Timestamp timestamp) const = 0; + // count of chunks virtual int64_t num_chunk() const = 0; @@ -119,6 +124,9 @@ class SegmentInternalInterface : public SegmentInterface { virtual int64_t size_per_chunk() const = 0; + virtual int64_t + get_active_count(Timestamp ts) const = 0; + protected: // internal API: return chunk_data in span virtual SpanBase diff --git a/internal/core/src/segcore/SegmentSealed.h b/internal/core/src/segcore/SegmentSealed.h index f5f0212f51bf5301e37117d40b86dc80fb1b5d5d..bd7dc6da7f7f99f0a798682fdb1776bb51eb3269 100644 --- a/internal/core/src/segcore/SegmentSealed.h +++ b/internal/core/src/segcore/SegmentSealed.h @@ -12,6 +12,7 @@ #include #include "segcore/SegmentInterface.h" +#include "pb/segcore.pb.h" #include "common/LoadInfo.h" #include @@ -22,6 +23,8 @@ class SegmentSealed : public SegmentInternalInterface { virtual void LoadIndex(const LoadIndexInfo& info) = 0; virtual void + LoadSegmentMeta(const milvus::proto::segcore::LoadSegmentMeta& meta) = 0; + virtual void LoadFieldData(const LoadFieldDataInfo& info) = 0; virtual void DropIndex(const FieldId field_id) = 0; diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index fdf35b71c5144ece6a30f7c2a247042c297aa2c8..4bd21f8398c8c999a9a5d519c3110712dc62486d 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -68,29 +68,51 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) { if (SystemProperty::Instance().IsSystem(field_id)) { auto system_field_type = SystemProperty::Instance().GetSystemFieldType(field_id); - Assert(system_field_type == SystemFieldType::RowId); - auto src_ptr = reinterpret_cast(info.blob); + if (system_field_type == SystemFieldType::Timestamp) { + auto src_ptr = reinterpret_cast(info.blob); + aligned_vector vec_data(info.row_count); + std::copy_n(src_ptr, info.row_count, vec_data.data()); + + auto size = info.row_count; + + // TODO: load from outside + TimestampIndex index; + auto min_slice_length = size < 4096 ? 1 : 4096; + auto meta = GenerateFakeSlices(src_ptr, size, min_slice_length); + index.set_length_meta(std::move(meta)); + index.build_with(src_ptr, size); + + // use special index + std::unique_lock lck(mutex_); + update_row_count(info.row_count); + AssertInfo(timestamps_.empty(), "already exists"); + timestamps_ = std::move(vec_data); + timestamp_index_ = std::move(index); - // prepare data - aligned_vector vec_data(info.row_count); - std::copy_n(src_ptr, info.row_count, vec_data.data()); + } else { + Assert(system_field_type == SystemFieldType::RowId); + auto src_ptr = reinterpret_cast(info.blob); - std::unique_ptr pk_index_; - // fix unintentional index update - if (schema_->get_is_auto_id()) { - pk_index_ = create_index(vec_data.data(), vec_data.size()); - } + // prepare data + aligned_vector vec_data(info.row_count); + std::copy_n(src_ptr, info.row_count, vec_data.data()); - // write data under lock - std::unique_lock lck(mutex_); - update_row_count(info.row_count); - AssertInfo(row_ids_.empty(), "already exists"); - row_ids_ = std::move(vec_data); + std::unique_ptr pk_index_; + // fix unintentional index update + if (schema_->get_is_auto_id()) { + pk_index_ = create_index(vec_data.data(), vec_data.size()); + } - if (schema_->get_is_auto_id()) { - primary_key_index_ = std::move(pk_index_); - } + // write data under lock + std::unique_lock lck(mutex_); + update_row_count(info.row_count); + AssertInfo(row_ids_.empty(), "already exists"); + row_ids_ = std::move(vec_data); + if (schema_->get_is_auto_id()) { + primary_key_index_ = std::move(pk_index_); + } + } ++system_ready_count_; } else { // prepare data @@ -194,8 +216,10 @@ SegmentSealedImpl::vector_search(int64_t vec_count, query::QueryInfo query_info, const void* query_data, int64_t query_count, + Timestamp timestamp, const BitsetView& bitset, QueryResult& output) const { + Assert(is_system_field_ready()); auto field_offset = query_info.field_offset_; auto& field_meta = schema_->operator[](field_offset); @@ -203,52 +227,54 @@ SegmentSealedImpl::vector_search(int64_t vec_count, if (get_bit(vecindex_ready_bitset_, field_offset)) { Assert(vecindexs_.is_ready(field_offset)); query::SearchOnSealed(*schema_, vecindexs_, query_info, query_data, query_count, bitset, output); - } else if (get_bit(field_data_ready_bitset_, field_offset)) { - query::dataset::QueryDataset dataset; - dataset.query_data = query_data; - dataset.num_queries = query_count; - // if(field_meta.is) - dataset.metric_type = query_info.metric_type_; - dataset.topk = query_info.topK_; - dataset.dim = field_meta.get_dim(); - - Assert(get_bit(field_data_ready_bitset_, field_offset)); - Assert(row_count_opt_.has_value()); - auto row_count = row_count_opt_.value(); - auto chunk_data = field_datas_[field_offset.get()].data(); - - auto sub_qr = [&] { - if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) { - return query::FloatSearchBruteForce(dataset, chunk_data, row_count, bitset); - } else { - return query::BinarySearchBruteForce(dataset, chunk_data, row_count, bitset); - } - }(); - - QueryResult results; - results.result_distances_ = std::move(sub_qr.mutable_values()); - results.internal_seg_offsets_ = std::move(sub_qr.mutable_labels()); - results.topK_ = dataset.topk; - results.num_queries_ = dataset.num_queries; - - output = std::move(results); - } else { + return; + } else if (!get_bit(field_data_ready_bitset_, field_offset)) { PanicInfo("Field Data is not loaded"); } + + query::dataset::QueryDataset dataset; + dataset.query_data = query_data; + dataset.num_queries = query_count; + // if(field_meta.is) + dataset.metric_type = query_info.metric_type_; + dataset.topk = query_info.topK_; + dataset.dim = field_meta.get_dim(); + + Assert(get_bit(field_data_ready_bitset_, field_offset)); + Assert(row_count_opt_.has_value()); + auto row_count = row_count_opt_.value(); + auto chunk_data = field_datas_[field_offset.get()].data(); + + auto sub_qr = [&] { + if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) { + return query::FloatSearchBruteForce(dataset, chunk_data, row_count, bitset); + } else { + return query::BinarySearchBruteForce(dataset, chunk_data, row_count, bitset); + } + }(); + + QueryResult results; + results.result_distances_ = std::move(sub_qr.mutable_values()); + results.internal_seg_offsets_ = std::move(sub_qr.mutable_labels()); + results.topK_ = dataset.topk; + results.num_queries_ = dataset.num_queries; + + output = std::move(results); } void SegmentSealedImpl::DropFieldData(const FieldId field_id) { if (SystemProperty::Instance().IsSystem(field_id)) { auto system_field_type = SystemProperty::Instance().GetSystemFieldType(field_id); - Assert(system_field_type == SystemFieldType::RowId); std::unique_lock lck(mutex_); --system_ready_count_; - auto row_ids = std::move(row_ids_); + if (system_field_type == SystemFieldType::RowId) { + auto row_ids = std::move(row_ids_); + } else if (system_field_type == SystemFieldType::Timestamp) { + auto ts = std::move(timestamps_); + } lck.unlock(); - - row_ids.clear(); } else { auto field_offset = schema_->get_offset(field_id); auto& field_meta = schema_->operator[](field_offset); @@ -280,7 +306,7 @@ SegmentSealedImpl::check_search(const query::Plan* plan) const { Assert(plan->extra_info_opt_.has_value()); if (!is_system_field_ready()) { - PanicInfo("System Field RowID is not loaded"); + PanicInfo("System Field RowID or Timestamp is not loaded"); } auto& request_fields = plan->extra_info_opt_.value().involved_fields_; @@ -429,6 +455,35 @@ SegmentSealedImpl::debug() const { return log_str; } +void +SegmentSealedImpl::LoadSegmentMeta(const proto::segcore::LoadSegmentMeta& segment_meta) { + std::unique_lock lck(mutex_); + std::vector slice_lengths; + for (auto& info : segment_meta.metas()) { + slice_lengths.push_back(info.row_count()); + } + timestamp_index_.set_length_meta(std::move(slice_lengths)); + PanicInfo("unimplemented"); +} +int64_t +SegmentSealedImpl::get_active_count(Timestamp ts) const { + // TODO optimize here to reduce expr search range + return this->get_row_count(); +} +void +SegmentSealedImpl::mask_with_timestamps(std::deque>& bitset_chunks, Timestamp timestamp) const { + // TODO change the + Assert(this->timestamps_.size() == get_row_count()); + Assert(bitset_chunks.size() == 1); + auto range = timestamp_index_.get_active_range(timestamp); + if (range.first == range.second && range.first == this->timestamps_.size()) { + // just skip + return; + } + auto mask = TimestampIndex::GenerateBitset(timestamp, range, this->timestamps_.data(), this->timestamps_.size()); + bitset_chunks[0] &= mask; +} + SegmentSealedPtr CreateSealedSegment(SchemaPtr schema) { return std::make_unique(schema); diff --git a/internal/core/src/segcore/SegmentSealedImpl.h b/internal/core/src/segcore/SegmentSealedImpl.h index e431aa73c8b46f5a0ba2c96fa3d16ea2be5ee6d5..8c375166ad0d2efb42bc8223d1e1724f92971fc3 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.h +++ b/internal/core/src/segcore/SegmentSealedImpl.h @@ -10,9 +10,11 @@ // or implied. See the License for the specific language governing permissions and limitations under the License #pragma once +#include #include "segcore/SegmentSealed.h" #include "SealedIndexingRecord.h" #include "ScalarIndex.h" +#include #include #include #include @@ -28,10 +30,11 @@ class SegmentSealedImpl : public SegmentSealed { void LoadFieldData(const LoadFieldDataInfo& info) override; void + LoadSegmentMeta(const milvus::proto::segcore::LoadSegmentMeta& segment_meta) override; + void DropIndex(const FieldId field_id) override; void DropFieldData(const FieldId field_id) override; - bool HasIndex(FieldId field_id) const override; bool @@ -82,6 +85,9 @@ class SegmentSealedImpl : public SegmentSealed { void check_search(const query::Plan* plan) const override; + int64_t + get_active_count(Timestamp ts) const override; + private: template static void @@ -100,17 +106,21 @@ class SegmentSealedImpl : public SegmentSealed { } } + void + mask_with_timestamps(std::deque>& bitset_chunks, Timestamp timestamp) const override; + void vector_search(int64_t vec_count, query::QueryInfo query_info, const void* query_data, int64_t query_count, + Timestamp timestamp, const BitsetView& bitset, QueryResult& output) const override; bool is_system_field_ready() const { - return system_ready_count_ == 1; + return system_ready_count_ == 2; } std::pair, std::vector> @@ -138,6 +148,8 @@ class SegmentSealedImpl : public SegmentSealed { SealedIndexingRecord vecindexs_; aligned_vector row_ids_; + aligned_vector timestamps_; + TimestampIndex timestamp_index_; SchemaPtr schema_; }; } // namespace milvus::segcore diff --git a/internal/core/src/segcore/TimestampIndex.cpp b/internal/core/src/segcore/TimestampIndex.cpp new file mode 100644 index 0000000000000000000000000000000000000000..84653249e7c35b6c9801a57136e2134f1e5d4b02 --- /dev/null +++ b/internal/core/src/segcore/TimestampIndex.cpp @@ -0,0 +1,104 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include + +namespace milvus::segcore { +void +TimestampIndex::set_length_meta(std::vector lengths) { + lengths_ = std::move(lengths); +} +void +TimestampIndex::build_with(const Timestamp* timestamps, int64_t size) { + auto num_slice = lengths_.size(); + Assert(num_slice > 0); + std::vector prefix_sums; + int offset = 0; + prefix_sums.push_back(offset); + std::vector timestamp_barriers; + Timestamp last_max_v = 0; + for (int slice_id = 0; slice_id < num_slice; ++slice_id) { + auto length = lengths_[slice_id]; + auto [min_v, max_v] = std::minmax_element(timestamps + offset, timestamps + offset + length); + Assert(last_max_v <= *min_v); + offset += length; + prefix_sums.push_back(offset); + timestamp_barriers.push_back(*min_v); + last_max_v = *max_v; + } + timestamp_barriers.push_back(last_max_v); + Assert(std::is_sorted(timestamp_barriers.begin(), timestamp_barriers.end())); + Assert(offset == size); + auto min_ts = timestamp_barriers[0]; + + this->size_ = size; + this->start_locs_ = std::move(prefix_sums); + this->min_timestamp_ = min_ts; + this->max_timestamp_ = last_max_v; + this->timestamp_barriers_ = std::move(timestamp_barriers); +} +std::pair +TimestampIndex::get_active_range(Timestamp query_timestamp) const { + if (query_timestamp >= max_timestamp_) { + // most common case + return {size_, size_}; + } + if (query_timestamp < min_timestamp_) { + return {0, 0}; + } + auto iter = std::upper_bound(timestamp_barriers_.begin(), timestamp_barriers_.end(), query_timestamp); + int block_id = (iter - timestamp_barriers_.begin()) - 1; + Assert(0 <= block_id && block_id < lengths_.size()); + return {start_locs_[block_id], start_locs_[block_id + 1]}; +} +boost::dynamic_bitset<> +TimestampIndex::GenerateBitset(Timestamp query_timestamp, + std::pair active_range, + const Timestamp* timestamps, + int64_t size) { + auto [beg, end] = active_range; + Assert(beg < end); + boost::dynamic_bitset<> bitset; + bitset.reserve(size); + bitset.resize(beg, true); + bitset.resize(size, false); + for (int64_t i = beg; i < end; ++i) { + bitset[i] = timestamps[i] <= query_timestamp; + } + return bitset; +} + +std::vector +GenerateFakeSlices(const Timestamp* timestamps, int64_t size, int min_slice_length) { + assert(min_slice_length >= 1); + std::vector results; + std::vector min_values(size); + Timestamp value = std::numeric_limits::max(); + for (int64_t i = 0; i < size; ++i) { + auto offset = size - 1 - i; + value = std::min(value, timestamps[offset]); + min_values[offset] = value; + } + value = std::numeric_limits::min(); + auto slice_length = 0; + for (int64_t i = 0; i < size; ++i) { + if (value <= min_values[i] && slice_length >= min_slice_length) { + // emit new slice + results.push_back(slice_length); + slice_length = 0; + } + value = std::max(value, timestamps[i]); + slice_length += 1; + } + results.push_back(slice_length); + return results; +} +} // namespace milvus::segcore diff --git a/internal/core/src/segcore/TimestampIndex.h b/internal/core/src/segcore/TimestampIndex.h new file mode 100644 index 0000000000000000000000000000000000000000..6b48e5b596ea876c188bd3304035be257f7e0bb9 --- /dev/null +++ b/internal/core/src/segcore/TimestampIndex.h @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once +#include +#include +#include +#include + +namespace milvus::segcore { +class TimestampIndex { + public: + void + set_length_meta(std::vector lengths); + + void + build_with(const Timestamp* timestamps, int64_t size); + // output bitset + + // Return range [beg, end) that is undecided + // [0, beg) shall be all OK + // [end, size) shall be all not OK + std::pair + get_active_range(Timestamp query_timestamp) const; + + static boost::dynamic_bitset<> + GenerateBitset(Timestamp query_timestamp, + std::pair active_range, + const Timestamp* timestamps, + int64_t size); + + private: + // numSlice + std::vector lengths_; + int64_t size_; + // numSlice + 1 + std::vector start_locs_; + Timestamp min_timestamp_; + Timestamp max_timestamp_; + // numSlice + 1 + std::vector timestamp_barriers_; +}; + +std::vector +GenerateFakeSlices(const Timestamp* timestamps, int64_t size, int min_slice_length = 1); + +} // namespace milvus::segcore diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index a6bd27bf41576c59113f4a4999ad040f09573cb0..2e7bda9ca0ae66738aaac3aae1d5529056345942 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -258,6 +258,19 @@ UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info } } +CStatus +LoadSealedSegmentMeta(CSegmentInterface c_segment, CProto LoadSegmentMetaProto) { + try { + auto segment_raw = (const milvus::segcore::SegmentGrowing*)c_segment; + auto segment = dynamic_cast(segment_raw); + + return milvus::SuccessCStatus(); + } catch (std::exception& e) { + // TODO + return milvus::FailureCStatus(UnexpectedError, e.what()); + } +} + int Close(CSegmentInterface c_segment) { auto segment = (milvus::segcore::SegmentGrowing*)c_segment; @@ -265,11 +278,6 @@ Close(CSegmentInterface c_segment) { return status.code(); } -int -BuildIndex(CCollection c_collection, CSegmentInterface c_segment) { - PanicInfo("unimplemented"); -} - bool IsOpened(CSegmentInterface c_segment) { auto segment = (milvus::segcore::SegmentGrowing*)c_segment; diff --git a/internal/core/src/segcore/segment_c.h b/internal/core/src/segcore/segment_c.h index df52c87b6fa6444fe95593699c47b0cfc9765414..1c96cfc98e31130b5af53cec6a04f220bd1a261e 100644 --- a/internal/core/src/segcore/segment_c.h +++ b/internal/core/src/segcore/segment_c.h @@ -103,9 +103,6 @@ UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info int Close(CSegmentInterface c_segment); -int -BuildIndex(CCollection c_collection, CSegmentInterface c_segment); - bool IsOpened(CSegmentInterface c_segment); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 018a3ac901b3634580393e17c548d7e0de3f8795..9ebc65bee4f7b7b2485728682d0411e43751b4ad 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -36,6 +36,7 @@ set(MILVUS_TEST_FILES test_init.cpp test_plan_proto.cpp test_get_entity_by_ids.cpp + test_timestamp_index.cpp ) add_executable(all_tests diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index a1163ad2579b936020c751121dd24f473f248a77..05d989e7fc88117ded40200f94527a9ce573f9d1 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -823,7 +823,7 @@ TEST(CApiTest, LoadIndex_Search) { constexpr auto DIM = 16; constexpr auto K = 10; - auto N = 1024 * 1024 * 10; + auto N = 1024 * 1024; auto num_query = 100; auto [raw_data, timestamps, uids] = generate_data(N); auto indexing = std::make_shared(); @@ -2569,6 +2569,14 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) { status = LoadFieldData(segment, c_id_field_data); assert(status.error_code == Success); + auto c_ts_field_data = CLoadFieldDataInfo{ + 1, + counter_col.data(), + N, + }; + status = LoadFieldData(segment, c_ts_field_data); + assert(status.error_code == Success); + status = UpdateSealedSegmentIndex(segment, c_load_index_info); assert(status.error_code == Success); @@ -2728,6 +2736,14 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) { status = LoadFieldData(segment, c_id_field_data); assert(status.error_code == Success); + auto c_ts_field_data = CLoadFieldDataInfo{ + 1, + counter_col.data(), + N, + }; + status = LoadFieldData(segment, c_ts_field_data); + assert(status.error_code == Success); + status = UpdateSealedSegmentIndex(segment, c_load_index_info); assert(status.error_code == Success); diff --git a/internal/core/unittest/test_expr.cpp b/internal/core/unittest/test_expr.cpp index 3d74c7f5d409c1aa7ea1cfd5632030721e997245..8d5b2eb974cd73f57d7a110ea90abb7c82f346b6 100644 --- a/internal/core/unittest/test_expr.cpp +++ b/internal/core/unittest/test_expr.cpp @@ -307,7 +307,7 @@ TEST(Expr, TestRange) { } auto seg_promote = dynamic_cast(seg.get()); - ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count()); + ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP); for (auto [clause, ref_func] : testcases) { auto loc = dsl_string_tmp.find("@@@@"); auto dsl_string = dsl_string_tmp; @@ -391,7 +391,7 @@ TEST(Expr, TestTerm) { } auto seg_promote = dynamic_cast(seg.get()); - ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count()); + ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP); for (auto [clause, ref_func] : testcases) { auto loc = dsl_string_tmp.find("@@@@"); auto dsl_string = dsl_string_tmp; @@ -493,7 +493,7 @@ TEST(Expr, TestSimpleDsl) { } auto seg_promote = dynamic_cast(seg.get()); - ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count()); + ExecExprVisitor visitor(*seg_promote, seg_promote->get_row_count(), MAX_TIMESTAMP); for (auto [clause, ref_func] : testcases) { Json dsl; dsl["bool"] = clause; diff --git a/internal/core/unittest/test_span.cpp b/internal/core/unittest/test_span.cpp index 9c21e45d0b0a0d61ba653bab23c31e80a1355f5b..c723a481bebf6dc19f743a99e0bfdab2d30bdba2 100644 --- a/internal/core/unittest/test_span.cpp +++ b/internal/core/unittest/test_span.cpp @@ -44,15 +44,15 @@ TEST(Span, Naive) { auto float_span = interface.chunk_data(FieldOffset(2), chunk_id); auto begin = chunk_id * size_per_chunk; auto end = std::min((chunk_id + 1) * size_per_chunk, N); - auto size_per_chunk = end - begin; - for (int i = 0; i < size_per_chunk * 512 / 8; ++i) { + auto size_of_chunk = end - begin; + for (int i = 0; i < size_of_chunk * 512 / 8; ++i) { ASSERT_EQ(vec_span.data()[i], vec_ptr[i + begin * 512 / 8]); } - for (int i = 0; i < size_per_chunk; ++i) { + for (int i = 0; i < size_of_chunk; ++i) { ASSERT_EQ(age_span.data()[i], age_ptr[i + begin]); } - for (int i = 0; i < size_per_chunk; ++i) { + for (int i = 0; i < size_of_chunk; ++i) { ASSERT_EQ(float_span.data()[i], float_ptr[i + begin * 32]); } } -} \ No newline at end of file +} diff --git a/internal/core/unittest/test_timestamp_index.cpp b/internal/core/unittest/test_timestamp_index.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a36b7676ef84b508b8fb2a2a5c99665678d77206 --- /dev/null +++ b/internal/core/unittest/test_timestamp_index.cpp @@ -0,0 +1,36 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include "utils/tools.h" +#include "test_utils/DataGen.h" +#include "segcore/TimestampIndex.h" +#include + +using namespace milvus; +using namespace milvus::segcore; +TEST(TimestampIndex, Naive) { + SUCCEED(); + std::vector timestamps{ + 1, 2, 14, 11, 13, 22, 21, 20, + }; + std::vector lengths = {2, 3, 3}; + TimestampIndex index; + index.set_length_meta(lengths); + index.build_with(timestamps.data(), timestamps.size()); + int x = 1 + 1; + + auto guessed_slice = GenerateFakeSlices(timestamps.data(), timestamps.size(), 2); + ASSERT_EQ(guessed_slice.size(), lengths.size()); + for (auto i = 0; i < lengths.size(); ++i) { + ASSERT_EQ(guessed_slice[i], lengths[i]); + } +} diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index 98a30b11cf75fb194a190540631988bfe62c4f4b..9d54e16c2139d8d482ee226711f461c1371aad73 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -299,6 +299,13 @@ SealedLoader(const GeneratedData& dataset, SegmentSealed& seg) { info.field_id = 0; // field id for RowId seg.LoadFieldData(info); } + { + LoadFieldDataInfo info; + info.blob = dataset.timestamps_.data(); + info.row_count = dataset.timestamps_.size(); + info.field_id = 1; + seg.LoadFieldData(info); + } int field_offset = 0; for (auto& meta : seg.get_schema().get_fields()) { LoadFieldDataInfo info; diff --git a/internal/proto/segcore.proto b/internal/proto/segcore.proto index ff3afbc30f936916a21cad750167b9e0f6c0cf77..bac5e69c24a6ac2744716a4a26a0294f39ac9533 100644 --- a/internal/proto/segcore.proto +++ b/internal/proto/segcore.proto @@ -14,3 +14,14 @@ message RetrieveResults { repeated schema.FieldData fields_data = 2; } +message LoadFieldMeta { + int64 min_timestamp = 1; + int64 max_timestamp = 2; + int64 row_count = 3; +} + +message LoadSegmentMeta { + // TODOs + repeated LoadFieldMeta metas = 1; + int64 total_size = 2; +} diff --git a/internal/proto/segcorepb/segcore.pb.go b/internal/proto/segcorepb/segcore.pb.go index f38a0007dbbe8ad363fbc2da3690994a46b97d5a..2364028ad322bc37d44968e400d165920629e69f 100644 --- a/internal/proto/segcorepb/segcore.pb.go +++ b/internal/proto/segcorepb/segcore.pb.go @@ -115,27 +115,139 @@ func (m *RetrieveResults) GetFieldsData() []*schemapb.FieldData { return nil } +type LoadFieldMeta struct { + MinTimestamp int64 `protobuf:"varint,1,opt,name=min_timestamp,json=minTimestamp,proto3" json:"min_timestamp,omitempty"` + MaxTimestamp int64 `protobuf:"varint,2,opt,name=max_timestamp,json=maxTimestamp,proto3" json:"max_timestamp,omitempty"` + RowCount int64 `protobuf:"varint,3,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadFieldMeta) Reset() { *m = LoadFieldMeta{} } +func (m *LoadFieldMeta) String() string { return proto.CompactTextString(m) } +func (*LoadFieldMeta) ProtoMessage() {} +func (*LoadFieldMeta) Descriptor() ([]byte, []int) { + return fileDescriptor_1d79fce784797357, []int{2} +} + +func (m *LoadFieldMeta) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadFieldMeta.Unmarshal(m, b) +} +func (m *LoadFieldMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadFieldMeta.Marshal(b, m, deterministic) +} +func (m *LoadFieldMeta) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadFieldMeta.Merge(m, src) +} +func (m *LoadFieldMeta) XXX_Size() int { + return xxx_messageInfo_LoadFieldMeta.Size(m) +} +func (m *LoadFieldMeta) XXX_DiscardUnknown() { + xxx_messageInfo_LoadFieldMeta.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadFieldMeta proto.InternalMessageInfo + +func (m *LoadFieldMeta) GetMinTimestamp() int64 { + if m != nil { + return m.MinTimestamp + } + return 0 +} + +func (m *LoadFieldMeta) GetMaxTimestamp() int64 { + if m != nil { + return m.MaxTimestamp + } + return 0 +} + +func (m *LoadFieldMeta) GetRowCount() int64 { + if m != nil { + return m.RowCount + } + return 0 +} + +type LoadSegmentMeta struct { + // TODOs + Metas []*LoadFieldMeta `protobuf:"bytes,1,rep,name=metas,proto3" json:"metas,omitempty"` + TotalSize int64 `protobuf:"varint,2,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadSegmentMeta) Reset() { *m = LoadSegmentMeta{} } +func (m *LoadSegmentMeta) String() string { return proto.CompactTextString(m) } +func (*LoadSegmentMeta) ProtoMessage() {} +func (*LoadSegmentMeta) Descriptor() ([]byte, []int) { + return fileDescriptor_1d79fce784797357, []int{3} +} + +func (m *LoadSegmentMeta) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadSegmentMeta.Unmarshal(m, b) +} +func (m *LoadSegmentMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadSegmentMeta.Marshal(b, m, deterministic) +} +func (m *LoadSegmentMeta) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadSegmentMeta.Merge(m, src) +} +func (m *LoadSegmentMeta) XXX_Size() int { + return xxx_messageInfo_LoadSegmentMeta.Size(m) +} +func (m *LoadSegmentMeta) XXX_DiscardUnknown() { + xxx_messageInfo_LoadSegmentMeta.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadSegmentMeta proto.InternalMessageInfo + +func (m *LoadSegmentMeta) GetMetas() []*LoadFieldMeta { + if m != nil { + return m.Metas + } + return nil +} + +func (m *LoadSegmentMeta) GetTotalSize() int64 { + if m != nil { + return m.TotalSize + } + return 0 +} + func init() { proto.RegisterType((*RetrieveRequest)(nil), "milvus.proto.segcore.RetrieveRequest") proto.RegisterType((*RetrieveResults)(nil), "milvus.proto.segcore.RetrieveResults") + proto.RegisterType((*LoadFieldMeta)(nil), "milvus.proto.segcore.LoadFieldMeta") + proto.RegisterType((*LoadSegmentMeta)(nil), "milvus.proto.segcore.LoadSegmentMeta") } func init() { proto.RegisterFile("segcore.proto", fileDescriptor_1d79fce784797357) } var fileDescriptor_1d79fce784797357 = []byte{ - // 222 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0xb1, 0x4b, 0xc4, 0x30, - 0x14, 0xc6, 0x39, 0x0f, 0x04, 0xd3, 0x3b, 0x84, 0xe2, 0x50, 0x1c, 0xe4, 0x38, 0x97, 0x43, 0x30, - 0x81, 0x2a, 0xae, 0x82, 0x14, 0xc1, 0x35, 0xa3, 0x4b, 0x49, 0xda, 0x67, 0x1b, 0x48, 0x9b, 0x9a, - 0xf7, 0xd2, 0xd1, 0xbf, 0x5d, 0x9a, 0x74, 0x50, 0x70, 0x71, 0xfb, 0xf2, 0x25, 0xbf, 0xef, 0x07, - 0x61, 0x7b, 0x84, 0xae, 0x71, 0x1e, 0xf8, 0xe4, 0x1d, 0xb9, 0xfc, 0x6a, 0x30, 0x76, 0x0e, 0x98, - 0x4e, 0x7c, 0xbd, 0xbb, 0xde, 0x61, 0xd3, 0xc3, 0xa0, 0x52, 0x7b, 0xd4, 0xec, 0x52, 0x02, 0x79, - 0x03, 0x33, 0x48, 0xf8, 0x0c, 0x80, 0x94, 0xdf, 0xb1, 0xad, 0x69, 0xb1, 0xd8, 0x1c, 0x36, 0xa7, - 0xac, 0x2c, 0xf8, 0xef, 0x91, 0xc4, 0xbe, 0x55, 0x28, 0x97, 0x47, 0xf9, 0x2d, 0xdb, 0xbb, 0x40, - 0x53, 0xa0, 0xfa, 0xc3, 0x80, 0x6d, 0xb1, 0x38, 0x3b, 0x6c, 0x4f, 0x17, 0x72, 0x97, 0xca, 0xd7, - 0xd8, 0x1d, 0xbf, 0x7e, 0x3a, 0x30, 0x58, 0xc2, 0x7f, 0x39, 0x9e, 0x59, 0x96, 0xc6, 0xeb, 0x56, - 0x91, 0x8a, 0x86, 0xac, 0xbc, 0xf9, 0x93, 0x89, 0xc2, 0x4a, 0x91, 0x92, 0x2c, 0x21, 0x4b, 0x7e, - 0x79, 0x7a, 0x7f, 0xec, 0x0c, 0xf5, 0x41, 0xf3, 0xc6, 0x0d, 0x22, 0x71, 0xf7, 0xc6, 0xad, 0x49, - 0x98, 0x91, 0xc0, 0x8f, 0xca, 0x8a, 0x38, 0x25, 0xd6, 0x7f, 0x9a, 0xb4, 0x3e, 0x8f, 0xc5, 0xc3, - 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xed, 0xbc, 0x2b, 0xca, 0x57, 0x01, 0x00, 0x00, + // 335 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0xc1, 0x4b, 0xf3, 0x30, + 0x18, 0xc6, 0xd9, 0xca, 0xf7, 0xe1, 0xb2, 0x8d, 0x41, 0xf1, 0x50, 0x14, 0x65, 0x74, 0x97, 0x21, + 0xd8, 0xc2, 0x14, 0xc1, 0x93, 0xa0, 0x43, 0x10, 0xf4, 0x92, 0x79, 0xf2, 0x52, 0xd2, 0xf6, 0x75, + 0x0b, 0x36, 0x4d, 0x6d, 0xde, 0x74, 0x63, 0x07, 0xff, 0x76, 0x49, 0x52, 0x71, 0x83, 0x5d, 0xbc, + 0x25, 0x4f, 0x9e, 0xe7, 0xfd, 0x3d, 0x2f, 0x21, 0x43, 0x05, 0xcb, 0x4c, 0xd6, 0x10, 0x55, 0xb5, + 0x44, 0xe9, 0x1f, 0x0b, 0x5e, 0x34, 0x5a, 0xb9, 0x5b, 0xd4, 0xbe, 0x9d, 0x0c, 0x54, 0xb6, 0x02, + 0xc1, 0x9c, 0x1a, 0xa6, 0x64, 0x44, 0x01, 0x6b, 0x0e, 0x0d, 0x50, 0xf8, 0xd4, 0xa0, 0xd0, 0xbf, + 0x20, 0x1e, 0xcf, 0x55, 0xd0, 0x19, 0x77, 0xa6, 0xfd, 0x59, 0x10, 0xed, 0x0f, 0x71, 0xd9, 0xa7, + 0xb9, 0xa2, 0xc6, 0xe4, 0x4f, 0xc8, 0x50, 0x6a, 0xac, 0x34, 0x26, 0xef, 0x1c, 0x8a, 0x5c, 0x05, + 0xdd, 0xb1, 0x37, 0xed, 0xd1, 0x81, 0x13, 0x1f, 0xad, 0x16, 0x7e, 0xed, 0x32, 0x94, 0x2e, 0x50, + 0xfd, 0x89, 0x71, 0x47, 0xfa, 0x6e, 0x78, 0x92, 0x33, 0x64, 0x96, 0xd0, 0x9f, 0x9d, 0x1f, 0xcc, + 0x58, 0xe0, 0x9c, 0x21, 0xa3, 0xc4, 0x45, 0xcc, 0x39, 0x6c, 0xc8, 0xf0, 0x59, 0xb2, 0xdc, 0x3e, + 0xbe, 0x00, 0x32, 0xd3, 0x5a, 0xf0, 0x32, 0x41, 0x2e, 0x40, 0x21, 0x13, 0x95, 0xed, 0xe1, 0xd1, + 0x81, 0xe0, 0xe5, 0xeb, 0x8f, 0x66, 0x4d, 0x6c, 0xb3, 0x63, 0xea, 0xb6, 0x26, 0xb6, 0xf9, 0x35, + 0x9d, 0x92, 0x5e, 0x2d, 0xd7, 0x49, 0x26, 0x75, 0x89, 0x81, 0x67, 0x0d, 0x47, 0xb5, 0x5c, 0x3f, + 0x98, 0x7b, 0xf8, 0x41, 0x46, 0x86, 0xbb, 0x80, 0xa5, 0x80, 0x12, 0x2d, 0xf9, 0x96, 0xfc, 0x13, + 0x80, 0xcc, 0x6c, 0x6e, 0xb6, 0x98, 0x44, 0x87, 0xbe, 0x28, 0xda, 0x6b, 0x4b, 0x5d, 0xc2, 0x3f, + 0x23, 0x04, 0x25, 0xb2, 0x22, 0x51, 0x7c, 0x0b, 0x6d, 0x99, 0x9e, 0x55, 0x16, 0x7c, 0x0b, 0xf7, + 0x37, 0x6f, 0xd7, 0x4b, 0x8e, 0x2b, 0x9d, 0x46, 0x99, 0x14, 0xb1, 0x1b, 0x7b, 0xc9, 0x65, 0x7b, + 0x8a, 0x79, 0x89, 0x50, 0x97, 0xac, 0x88, 0x2d, 0x29, 0x6e, 0x49, 0x55, 0x9a, 0xfe, 0xb7, 0xc2, + 0xd5, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xac, 0x78, 0x16, 0xc2, 0x3c, 0x02, 0x00, 0x00, } diff --git a/internal/querynode/index.go b/internal/querynode/index.go deleted file mode 100644 index 1ec876746782040513779e941bce9fc300ccf99e..0000000000000000000000000000000000000000 --- a/internal/querynode/index.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -package querynode - -/* - -#cgo CFLAGS: -I../core/output/include - -#cgo LDFLAGS: -L../core/output/lib -lmilvus_segcore -Wl,-rpath=../core/output/lib - -#include "segcore/collection_c.h" -#include "segcore/segment_c.h" - -*/ -import "C" -import ( - "github.com/milvus-io/milvus/internal/proto/commonpb" -) - -type IndexConfig struct{} - -func (s *Segment) buildIndex(collection *Collection) commonpb.Status { - /* - int - BuildIndex(CCollection c_collection, CSegmentBase c_segment); - */ - var status = C.BuildIndex(collection.collectionPtr, s.segmentPtr) - if status != 0 { - return commonpb.Status{ErrorCode: commonpb.ErrorCode_BuildIndexError} - } - return commonpb.Status{ErrorCode: commonpb.ErrorCode_Success} -} - -func (s *Segment) dropIndex(fieldID int64) commonpb.Status { - // WARN: Not support yet - - return commonpb.Status{ErrorCode: commonpb.ErrorCode_Success} -} diff --git a/internal/querynode/segment_loader.go b/internal/querynode/segment_loader.go index 640bbc2c1b38543e4f4524e8ef5c6019ed18fb43..0013f24e80fa69c0bd70fc87e2d724091df2da4f 100644 --- a/internal/querynode/segment_loader.go +++ b/internal/querynode/segment_loader.go @@ -227,10 +227,6 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths blobs := make([]*storage.Blob, 0) for _, binlogPath := range binlogPaths { fieldID := binlogPath.FieldID - if fieldID == timestampFieldID { - // seg core doesn't need timestamp field - continue - } paths := binlogPath.Binlogs log.Debug("load segment fields data", @@ -254,6 +250,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths } _, _, insertData, err := iCodec.Deserialize(blobs) + if err != nil { log.Error(err.Error()) return err