diff --git a/internal/core/bench/bench_search.cpp b/internal/core/bench/bench_search.cpp index cd581e07fc9f3a70d6b6b3d7a5a90bb0618c1768..26b4eb3491924477831c3ab03d014778176b2920 100644 --- a/internal/core/bench/bench_search.cpp +++ b/internal/core/bench/bench_search.cpp @@ -69,7 +69,9 @@ Search_SmallIndex(benchmark::State& state) { auto is_small_index = state.range(0); auto chunk_size = state.range(1) * 1024; - auto segment = CreateGrowingSegment(schema, chunk_size); + auto segconf = SegcoreConfig::default_config(); + segconf.set_size_per_chunk(chunk_size); + auto segment = CreateGrowingSegment(schema, segconf); if (!is_small_index) { segment->debug_disable_small_index(); } diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index f07553085e183bf5751e055050dd612ab6a14d95..6b867b18b42ae59746bcf486368b4ada03b1eda1 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -7,4 +7,4 @@ set(COMMON_SRC add_library(milvus_common ${COMMON_SRC} ) -target_link_libraries(milvus_common milvus_proto) +target_link_libraries(milvus_common milvus_proto yaml-cpp) diff --git a/internal/core/src/common/SystemProperty.cpp b/internal/core/src/common/SystemProperty.cpp index e6fc4e29f425116e3e4c9dcbf08d1e61209457dd..2d7902c91467af69701c6f3e0f6f2727b0738451 100644 --- a/internal/core/src/common/SystemProperty.cpp +++ b/internal/core/src/common/SystemProperty.cpp @@ -11,6 +11,8 @@ #include "common/SystemProperty.h" #include "utils/EasyAssert.h" +#include +#include "common/Types.h" namespace milvus { class SystemPropertyImpl : public SystemProperty { @@ -50,19 +52,20 @@ class SystemPropertyImpl : public SystemProperty { return name_to_types_.count(field_name); } - friend const SystemProperty& - SystemProperty::Instance(); + friend SystemPropertyImpl& + InstanceImpl(); private: std::map name_to_types_; std::map id_to_types_; }; -const SystemProperty& -SystemProperty::Instance() { +SystemPropertyImpl& +InstanceImpl() { static auto impl = [] { SystemPropertyImpl impl; using Type = SystemFieldType; + impl.name_to_types_.emplace(FieldName("RowID"), Type::RowId); impl.id_to_types_.emplace(FieldId(0), Type::RowId); @@ -73,4 +76,10 @@ SystemProperty::Instance() { }(); return impl; } + +const SystemProperty& +SystemProperty::Instance() { + return InstanceImpl(); +} + }; // namespace milvus diff --git a/internal/core/src/common/SystemProperty.h b/internal/core/src/common/SystemProperty.h index 4dd8d7b5354023c2c7d2ffb1333e8c549a68b1db..6c45e8ecbe2b118cb291a4295b37b86816523a41 100644 --- a/internal/core/src/common/SystemProperty.h +++ b/internal/core/src/common/SystemProperty.h @@ -11,6 +11,8 @@ #pragma once #include "common/Types.h" +#include "utils/Json.h" +#include namespace milvus { @@ -22,8 +24,20 @@ enum class SystemFieldType { class SystemProperty { public: + SystemProperty() = default; + ~SystemProperty() = default; + static const SystemProperty& Instance(); + SystemProperty(const SystemProperty&) = delete; + + SystemProperty& + operator=(const SystemProperty&) = delete; + + protected: + SystemProperty(SystemProperty&&) = default; + SystemProperty& + operator=(SystemProperty&&) = default; public: virtual bool @@ -42,4 +56,7 @@ class SystemProperty { IsSystem(FieldName field_name) const = 0; }; +void +InitalizeSegcore(const std::string& segcore_yaml_path); + } // namespace milvus diff --git a/internal/core/src/query/SearchOnGrowing.cpp b/internal/core/src/query/SearchOnGrowing.cpp index 1e6f1c7252b159e71c2dd10c42b7e0e57d24166d..e043731931132afd15afa9cbc9d23a274d308263 100644 --- a/internal/core/src/query/SearchOnGrowing.cpp +++ b/internal/core/src/query/SearchOnGrowing.cpp @@ -77,7 +77,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment, if (indexing_record.is_in(vecfield_offset)) { auto max_indexed_id = indexing_record.get_finished_ack(); const auto& field_indexing = indexing_record.get_vec_field_indexing(vecfield_offset); - auto search_conf = field_indexing.get_search_conf(topK); + auto search_conf = field_indexing.get_search_params(topK); Assert(vec_ptr->get_size_per_chunk() == field_indexing.get_size_per_chunk()); for (int chunk_id = current_chunk_id; chunk_id < max_indexed_id; ++chunk_id) { diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index fc796361d1d8dcfa905ff5a79c5123da961ea0aa..10ed510ca85f84fd06e610cbba4d7d450e587cd8 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -14,7 +14,7 @@ set(SEGCORE_FILES load_index_c.cpp SealedIndexingRecord.cpp SegmentInterface.cpp - ) + SegcoreConfig.h SegcoreConfig.cpp) add_library(milvus_segcore SHARED ${SEGCORE_FILES} ) diff --git a/internal/core/src/segcore/FieldIndexing.cpp b/internal/core/src/segcore/FieldIndexing.cpp index 57c7654c85b8372878dfcafec0f4be0d60f27f34..b44a6594d18a70316d04b8702c75943d9dfea905 100644 --- a/internal/core/src/segcore/FieldIndexing.cpp +++ b/internal/core/src/segcore/FieldIndexing.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include "common/SystemProperty.h" namespace milvus::segcore { void @@ -24,7 +26,7 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec Assert(source); auto num_chunk = source->num_chunk(); assert(ack_end <= num_chunk); - auto conf = get_build_conf(); + auto conf = get_build_params(); data_.grow_to_at_least(ack_end); for (int chunk_id = ack_beg; chunk_id < ack_end; chunk_id++) { const auto& chunk = source->get_chunk(chunk_id); @@ -38,27 +40,37 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec } knowhere::Config -VectorFieldIndexing::get_build_conf() const { +VectorFieldIndexing::get_build_params() const { // TODO auto type_opt = field_meta_.get_metric_type(); Assert(type_opt.has_value()); - auto type_name = MetricTypeToName(type_opt.value()); - return knowhere::Config{{knowhere::meta::DIM, field_meta_.get_dim()}, - {knowhere::IndexParams::nlist, 100}, - {knowhere::IndexParams::nprobe, 4}, - {knowhere::Metric::TYPE, type_name}, - {knowhere::meta::DEVICEID, 0}}; + auto metric_type = type_opt.value(); + auto type_name = MetricTypeToName(metric_type); + auto& config = segcore_config_.at(metric_type); + auto base_params = config.build_params; + + Assert(base_params.count("nlist")); + base_params[knowhere::meta::DIM] = field_meta_.get_dim(); + base_params[knowhere::Metric::TYPE] = type_name; + + return base_params; } knowhere::Config -VectorFieldIndexing::get_search_conf(int top_K) const { +VectorFieldIndexing::get_search_params(int top_K) const { // TODO auto type_opt = field_meta_.get_metric_type(); Assert(type_opt.has_value()); - auto type_name = MetricTypeToName(type_opt.value()); - return knowhere::Config{{knowhere::meta::DIM, field_meta_.get_dim()}, {knowhere::meta::TOPK, top_K}, - {knowhere::IndexParams::nlist, 100}, {knowhere::IndexParams::nprobe, 4}, - {knowhere::Metric::TYPE, type_name}, {knowhere::meta::DEVICEID, 0}}; + auto metric_type = type_opt.value(); + auto type_name = MetricTypeToName(metric_type); + auto& config = segcore_config_.at(metric_type); + + auto base_params = config.search_params; + Assert(base_params.count("nprobe")); + base_params[knowhere::meta::TOPK] = top_K; + base_params[knowhere::Metric::TYPE] = type_name; + + return base_params; } void @@ -103,10 +115,10 @@ ScalarFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const } std::unique_ptr -CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk) { +CreateIndex(const FieldMeta& field_meta, const SegcoreConfig& segcore_config) { if (field_meta.is_vector()) { if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) { - return std::make_unique(field_meta, size_per_chunk); + return std::make_unique(field_meta, segcore_config); } else { // TODO PanicInfo("unsupported"); @@ -114,19 +126,19 @@ CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk) { } switch (field_meta.get_data_type()) { case DataType::BOOL: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::INT8: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::INT16: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::INT32: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::INT64: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::FLOAT: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); case DataType::DOUBLE: - return std::make_unique>(field_meta, size_per_chunk); + return std::make_unique>(field_meta, segcore_config); default: PanicInfo("unsupported"); } diff --git a/internal/core/src/segcore/FieldIndexing.h b/internal/core/src/segcore/FieldIndexing.h index be15ff556271267df78f4b32ab329a531ac422eb..bb7f41c51e9fafb0df0b52162992b5277871c08f 100644 --- a/internal/core/src/segcore/FieldIndexing.h +++ b/internal/core/src/segcore/FieldIndexing.h @@ -19,6 +19,7 @@ #include "InsertRecord.h" #include #include +#include "segcore/SegcoreConfig.h" namespace milvus::segcore { @@ -26,8 +27,8 @@ namespace milvus::segcore { // All concurrent class FieldIndexing { public: - explicit FieldIndexing(const FieldMeta& field_meta, int64_t size_per_chunk) - : field_meta_(field_meta), size_per_chunk_(size_per_chunk) { + explicit FieldIndexing(const FieldMeta& field_meta, const SegcoreConfig& segcore_config) + : field_meta_(field_meta), segcore_config_(segcore_config) { } FieldIndexing(const FieldIndexing&) = delete; FieldIndexing& @@ -44,7 +45,7 @@ class FieldIndexing { int64_t get_size_per_chunk() const { - return size_per_chunk_; + return segcore_config_.get_size_per_chunk(); } virtual knowhere::Index* @@ -53,7 +54,7 @@ class FieldIndexing { protected: // additional info const FieldMeta& field_meta_; - const int64_t size_per_chunk_; + const SegcoreConfig& segcore_config_; }; template class ScalarFieldIndexing : public FieldIndexing { @@ -89,21 +90,22 @@ class VectorFieldIndexing : public FieldIndexing { } knowhere::Config - get_build_conf() const; + get_build_params() const; + knowhere::Config - get_search_conf(int top_k) const; + get_search_params(int top_k) const; private: tbb::concurrent_vector> data_; }; std::unique_ptr -CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk); +CreateIndex(const FieldMeta& field_meta, const SegcoreConfig& segcore_config); class IndexingRecord { public: - explicit IndexingRecord(const Schema& schema, int64_t size_per_chunk) - : schema_(schema), size_per_chunk_(size_per_chunk) { + explicit IndexingRecord(const Schema& schema, const SegcoreConfig& segcore_config) + : schema_(schema), segcore_config_(segcore_config) { Initialize(); } @@ -125,7 +127,7 @@ class IndexingRecord { } } - field_indexings_.try_emplace(offset, CreateIndex(field, size_per_chunk_)); + field_indexings_.try_emplace(offset, CreateIndex(field, segcore_config_)); } assert(offset_id == schema_.size()); } @@ -170,6 +172,7 @@ class IndexingRecord { private: const Schema& schema_; + const SegcoreConfig& segcore_config_; private: // control info @@ -177,7 +180,6 @@ class IndexingRecord { // std::atomic finished_ack_ = 0; AckResponder finished_ack_; std::mutex mutex_; - int64_t size_per_chunk_; private: // field_offset => indexing diff --git a/internal/core/src/segcore/SegcoreConfig.cpp b/internal/core/src/segcore/SegcoreConfig.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e7808e9da8d6d86fcaa2a40652b15c0680c81a58 --- /dev/null +++ b/internal/core/src/segcore/SegcoreConfig.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include "common/Schema.h" +#include "SegcoreConfig.h" +#include "utils/Json.h" +#include "yaml-cpp/yaml.h" + +namespace milvus::segcore { + +static YAML::Node +subnode(const YAML::Node& parent, const std::string& key) { + AssertInfo(parent.IsMap(), "wrong type node when getting key[" + key + "]"); + auto& node = parent[key]; + AssertInfo(node.IsDefined(), "key[" + key + "] not found in sub-node"); + return node; +} + +template +std::vector +apply_parser(const YAML::Node& node, Func func) { + std::vector results; + Assert(node.IsDefined()); + if (node.IsScalar()) { + results.emplace_back(func(node)); + } else if (node.IsSequence()) { + for (auto& element : node) { + Assert(element.IsScalar()); + results.emplace_back(func(element)); + } + } else { + PanicInfo("node should be scalar or sequence"); + } + return results; +} + +SegcoreConfig +SegcoreConfig::parse_from(const std::string& config_path) { + try { + SegcoreConfig result; + YAML::Node top_config = YAML::LoadFile(config_path); + Assert(top_config.IsMap()); + auto seg_config = subnode(top_config, "segcore"); + auto chunk_size = subnode(seg_config, "chunk_size").as(); + result.size_per_chunk_ = chunk_size; + + auto index_list = subnode(seg_config, "small_index"); + + Assert(index_list.IsSequence()); + for (auto index : index_list) { + Assert(index.IsMap()); + auto metric_types = apply_parser(subnode(index, "metric_type"), [](const YAML::Node& node) { + return GetMetricType(node.as()); + }); + + { + std::sort(metric_types.begin(), metric_types.end()); + auto end_iter = std::unique(metric_types.begin(), metric_types.end()); + metric_types.resize(end_iter - metric_types.begin()); + } + + auto index_type = index["index_type"].as(); + AssertInfo(index_type == "IVF", "only ivf is supported now"); + + SmallIndexConf conf; + conf.index_type = index_type; + + // parse build config + for (auto node : index["build_params"]) { + // TODO: currently support IVF only + auto key = node.first.as(); + Assert(key == "nlist"); + auto value = node.second.as(); + conf.build_params[key] = value; + } + + // parse search config + for (auto node : index["search_params"]) { + // TODO: currently support IVF only + auto key = node.first.as(); + Assert(key == "nprobe"); + auto value = node.second.as(); + conf.search_params[key] = value; + } + + for (auto metric_type : metric_types) { + Assert(result.table_.count(metric_type)); + result.table_[metric_type] = conf; + } + } + } catch (const WrappedRuntimeError& e) { + // re-throw + throw e; + } catch (std::exception& e) { + PanicInfo(std::string("Invalid Yaml:\n") + e.what()); + } +} + +} // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegcoreConfig.h b/internal/core/src/segcore/SegcoreConfig.h new file mode 100644 index 0000000000000000000000000000000000000000..21cd0414a34325133814de9e48074e33b751ee00 --- /dev/null +++ b/internal/core/src/segcore/SegcoreConfig.h @@ -0,0 +1,73 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#pragma once +#include "common/Types.h" +#include "utils/Json.h" +#include +#include + +namespace milvus::segcore { + +struct SmallIndexConf { + std::string index_type; + nlohmann::json build_params; + nlohmann::json search_params; +}; + +class SegcoreConfig { + public: + static SegcoreConfig + parse_from(const std::string& string_path); + static SegcoreConfig + default_config() { + // TODO: remove this when go side is ready + SegcoreConfig config; + config.set_size_per_chunk(32 * 1024); + SmallIndexConf sub_conf; + sub_conf.build_params["nlist"] = 100; + sub_conf.search_params["nprobe"] = 4; + sub_conf.index_type = "IVF"; + config.table_[MetricType::METRIC_L2] = sub_conf; + config.table_[MetricType::METRIC_INNER_PRODUCT] = sub_conf; + return config; + } + + const SmallIndexConf& + at(MetricType metric_type) const { + Assert(table_.count(metric_type)); + return table_.at(metric_type); + } + + int64_t + get_size_per_chunk() const { + return size_per_chunk_; + } + + void + set_size_per_chunk(int64_t size_per_chunk) { + size_per_chunk_ = size_per_chunk; + } + + void + set_small_index_config(MetricType metric_type, const SmallIndexConf& small_index_conf) { + table_[metric_type] = small_index_conf; + } + + protected: + SegcoreConfig() = default; + + private: + int64_t size_per_chunk_ = -1; + std::map table_; +}; + +} // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentGrowing.cpp b/internal/core/src/segcore/SegmentGrowing.cpp index 0d51d8abb26fbc69398db419ef31e721c1b33214..bdc1fc6421c3b24b1cb6bdc6611898e9ca4217ae 100644 --- a/internal/core/src/segcore/SegmentGrowing.cpp +++ b/internal/core/src/segcore/SegmentGrowing.cpp @@ -20,8 +20,8 @@ TestABI() { } std::unique_ptr -CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk) { - auto segment = std::make_unique(schema, size_per_chunk); +CreateGrowingSegment(SchemaPtr schema, const SegcoreConfig& segcore_config) { + auto segment = std::make_unique(schema, segcore_config); return segment; } diff --git a/internal/core/src/segcore/SegmentGrowing.h b/internal/core/src/segcore/SegmentGrowing.h index 6cc5f11b11c0954fa1c154547bdb6fa75962cbf8..d24be928636d59f7bcabff4cfe7ef586ae1df17a 100644 --- a/internal/core/src/segcore/SegmentGrowing.h +++ b/internal/core/src/segcore/SegmentGrowing.h @@ -95,7 +95,13 @@ class SegmentGrowing : public SegmentInternalInterface { using SegmentGrowingPtr = std::unique_ptr; SegmentGrowingPtr -CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk = 32 * 1024); +CreateGrowingSegment(SchemaPtr schema, const SegcoreConfig& segcore_config); + +inline SegmentGrowingPtr +CreateGrowingSegment(SchemaPtr schema) { + auto seg_conf = SegcoreConfig::default_config(); + return CreateGrowingSegment(schema, seg_conf); +} } // namespace segcore } // namespace milvus diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index b6dd81795718c304e3451ae85cdb24dc4bcb6cc7..ae311d93d2ed695efa26eafa1e1e1a47e02a0759 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -190,7 +190,8 @@ SegmentGrowingImpl::do_insert(int64_t reserved_begin, } record_.ack_responder_.AddSegment(reserved_begin, reserved_begin + size); if (!debug_disable_small_index_) { - indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / size_per_chunk_, record_); + indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / segcore_config_.get_size_per_chunk(), + record_); } } @@ -241,9 +242,10 @@ SegmentGrowingImpl::Close() { int64_t SegmentGrowingImpl::GetMemoryUsageInBytes() const { int64_t total_bytes = 0; - int64_t ins_n = upper_align(record_.reserved, size_per_chunk_); + auto size_per_chunk = segcore_config_.get_size_per_chunk(); + int64_t ins_n = upper_align(record_.reserved, size_per_chunk); total_bytes += ins_n * (schema_->get_total_sizeof() + 16 + 1); - int64_t del_n = upper_align(deleted_record_.reserved, size_per_chunk_); + int64_t del_n = upper_align(deleted_record_.reserved, size_per_chunk); total_bytes += del_n * (16 * 2); return total_bytes; } @@ -268,7 +270,7 @@ SegmentGrowingImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) int64_t SegmentGrowingImpl::num_chunk() const { auto size = get_insert_record().ack_responder_.GetAck(); - return upper_div(size, size_per_chunk_); + return upper_div(size, segcore_config_.get_size_per_chunk()); } void SegmentGrowingImpl::vector_search(int64_t vec_count, diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index 7a62a8edee6e2e44d9debca35666f4865b207dc7..25ea5162e055af81635da4788a1e6b3dd100e702 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -108,7 +108,7 @@ class SegmentGrowingImpl : public SegmentGrowing { int64_t size_per_chunk() const final { - return size_per_chunk_; + return segcore_config_.get_size_per_chunk(); } public: @@ -157,13 +157,13 @@ class SegmentGrowingImpl : public SegmentGrowing { public: friend std::unique_ptr - CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk); + CreateGrowingSegment(SchemaPtr schema, const SegcoreConfig& segcore_config); - explicit SegmentGrowingImpl(SchemaPtr schema, int64_t size_per_chunk) - : size_per_chunk_(size_per_chunk), + explicit SegmentGrowingImpl(SchemaPtr schema, const SegcoreConfig& segcore_config) + : segcore_config_(segcore_config), schema_(std::move(schema)), - record_(*schema_, size_per_chunk), - indexing_record_(*schema_, size_per_chunk) { + record_(*schema_, segcore_config.get_size_per_chunk()), + indexing_record_(*schema_, segcore_config_) { } void @@ -199,7 +199,7 @@ class SegmentGrowingImpl : public SegmentGrowing { const std::vector>& columns_data); private: - int64_t size_per_chunk_; + SegcoreConfig segcore_config_; SchemaPtr schema_; std::atomic state_ = SegmentState::Open; diff --git a/internal/core/src/segcore/SegmentSealed.h b/internal/core/src/segcore/SegmentSealed.h index b532fe9cde495a32cf71f78b4c2b3a2f85d0a360..8afeadd59548565ef85c001c39d79568099c05c0 100644 --- a/internal/core/src/segcore/SegmentSealed.h +++ b/internal/core/src/segcore/SegmentSealed.h @@ -35,6 +35,6 @@ class SegmentSealed : public SegmentInternalInterface { using SegmentSealedPtr = std::unique_ptr; SegmentSealedPtr -CreateSealedSegment(SchemaPtr schema, int64_t size_per_chunk = 32 * 1024); +CreateSealedSegment(SchemaPtr schema); } // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index 769e3e086769fcf3e097955fca5d1f1f7e679df4..279352dfec26596ea87db8b3430015a419793955 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -384,7 +384,7 @@ SegmentSealedImpl::HasFieldData(FieldId field_id) const { } SegmentSealedPtr -CreateSealedSegment(SchemaPtr schema, int64_t size_per_chunk) { +CreateSealedSegment(SchemaPtr schema) { return std::make_unique(schema); } diff --git a/internal/core/src/utils/EasyAssert.cpp b/internal/core/src/utils/EasyAssert.cpp index ae74724f53cf045b5198f4ceaeb85ae42e7ef798..1bc4506d668fedd03571b33cb94b64447f6bd762 100644 --- a/internal/core/src/utils/EasyAssert.cpp +++ b/internal/core/src/utils/EasyAssert.cpp @@ -46,11 +46,11 @@ EasyAssertInfo( [[noreturn]] void ThrowWithTrace(const std::exception& exception) { - if (typeid(exception) == typeid(WrappedRuntimError)) { + if (typeid(exception) == typeid(WrappedRuntimeError)) { throw exception; } auto err_msg = exception.what() + std::string("\n") + EasyStackTrace(); - throw WrappedRuntimError(err_msg); + throw WrappedRuntimeError(err_msg); } } // namespace milvus::impl diff --git a/internal/core/src/utils/EasyAssert.h b/internal/core/src/utils/EasyAssert.h index 8ef54042d7530665686065c0306b8d41e1d8ae16..bf5f5ca729a8658e270f0c952250c4ebd3e90a75 100644 --- a/internal/core/src/utils/EasyAssert.h +++ b/internal/core/src/utils/EasyAssert.h @@ -18,19 +18,22 @@ /* Paste this on the file you want to debug. */ -namespace milvus::impl { +namespace milvus { +namespace impl { void EasyAssertInfo( bool value, std::string_view expr_str, std::string_view filename, int lineno, std::string_view extra_info); -class WrappedRuntimError : public std::runtime_error { - using std::runtime_error::runtime_error; -}; - [[noreturn]] void ThrowWithTrace(const std::exception& exception); -} // namespace milvus::impl +} // namespace impl + +class WrappedRuntimeError : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +} // namespace milvus #define AssertInfo(expr, info) \ do { \ diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 787d7e6535271096c6cd532ae70455d23aa8a607..7098c2e351ebede29503261bd77d85b8baf58eef 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -46,9 +46,7 @@ endif() # ****************************** Thirdparty yaml *************************************** -if ( MILVUS_WITH_YAMLCPP ) - add_subdirectory( yaml-cpp ) -endif() +add_subdirectory( yaml-cpp ) # ****************************** Thirdparty opentracing *************************************** if ( MILVUS_WITH_OPENTRACING ) diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 51bc740e6e0b054a4f78c1a2481bca73074e1da6..edf8669ba44ec2133b0a4c52191ac14e495b22d2 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -1,5 +1,9 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src) include_directories(${CMAKE_HOME_DIRECTORY}/src/index/knowhere) +include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) + +add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") + set(MILVUS_TEST_FILES test_naive.cpp test_segcore.cpp @@ -17,14 +21,18 @@ set(MILVUS_TEST_FILES test_interface.cpp test_span.cpp test_load.cpp + init_gtest.cpp ) + add_executable(all_tests ${MILVUS_TEST_FILES} ) # check if memory leak exists in index builder set(INDEX_BUILDER_TEST_FILES - test_index_wrapper.cpp) + test_index_wrapper.cpp + ) + add_executable(index_builder_test ${INDEX_BUILDER_TEST_FILES} ) @@ -39,7 +47,6 @@ target_link_libraries(index_builder_test target_link_libraries(all_tests gtest - gtest_main milvus_segcore milvus_indexbuilder log diff --git a/internal/core/unittest/init_gtest.cpp b/internal/core/unittest/init_gtest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dd6578404fb51cd0513844269e9bdd55b8789454 --- /dev/null +++ b/internal/core/unittest/init_gtest.cpp @@ -0,0 +1,24 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +#include +#include "common/SystemProperty.h" +#ifndef MILVUS_TEST_SEGCORE_YAML_PATH +#error MILVUS_TEST_SEGCORE_YAML_PATH is not defined +#define MILVUS_TEST_SEGCORE_YAML_PATH "" +#endif + +int +main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/internal/core/unittest/test_span.cpp b/internal/core/unittest/test_span.cpp index 8c582f344c97e878660f3f3f3899be517d2df32f..9c21e45d0b0a0d61ba653bab23c31e80a1355f5b 100644 --- a/internal/core/unittest/test_span.cpp +++ b/internal/core/unittest/test_span.cpp @@ -26,7 +26,8 @@ TEST(Span, Naive) { schema->AddDebugField("floatvec", DataType::VECTOR_FLOAT, 32, MetricType::METRIC_L2); auto dataset = DataGen(schema, N); - auto segment = CreateGrowingSegment(schema, size_per_chunk); + auto seg_conf = SegcoreConfig::default_config(); + auto segment = CreateGrowingSegment(schema, seg_conf); segment->PreInsert(N); segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_); auto vec_ptr = dataset.get_col(0); diff --git a/internal/core/unittest/test_utils/test_segcore.yaml b/internal/core/unittest/test_utils/test_segcore.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cdad64f22df812b5a0cef1efd641e0b179d35abb --- /dev/null +++ b/internal/core/unittest/test_utils/test_segcore.yaml @@ -0,0 +1,16 @@ +segcore: + chunk_size: 32768 + small_index: + - metric_type: ["L2", "IP"] + index_type: "IVF" + build_params: + nlist: 100 + search_params: + nprobe: 4 + # binary is not ready yet + - metric_type: ["jaccard"] + index_type: "IVF" + build_params: + nlist: 100 + search_params: + nprobe: 4