diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 72cf488cba01a4433ef776cc3e8259081c97224d..3f6b12cf1bcd15af14aa308bafcefc1f326be7e6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -40,6 +40,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-394 - Update scheduler unittest - MS-400 - Add timestamp record in task state change function - MS-402 - Add dump implementation for TaskTableItem +- MS-406 - Add table flag for meta - MS-403 - Add GpuCacheMgr - MS-404 - Release index after search task done avoid memory increment continues - MS-405 - Add delete task support diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index b143f0c23383a1ce6d11c6152b7486156127431e..f42154174166e4741c32bc2c0e57bf20e23a3f5c 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -29,6 +29,7 @@ public: virtual Status AllTables(std::vector& table_schema_array) = 0; virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; virtual Status PreloadTable(const std::string& table_id) = 0; + virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; virtual Status InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 8a002f29b094d2ec4c3d67c9ac88bd9a00a90cc6..6b0b87889ce208cf7b57208d2be729c23f2925ef 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -154,6 +154,10 @@ Status DBImpl::PreloadTable(const std::string &table_id) { return Status::OK(); } +Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + return meta_ptr_->UpdateTableFlag(table_id, flag); +} + Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { return meta_ptr_->Count(table_id, row_count); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 97c36fadbc14e0839f936336668313e34c1e98c1..65ac489c49ac3e6f9e9e658575b2e8692f04e16d 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -36,40 +36,32 @@ class DBImpl : public DB { explicit DBImpl(const Options &options); - Status - CreateTable(meta::TableSchema &table_schema) override; + Status CreateTable(meta::TableSchema &table_schema) override; - Status - DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; + Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; - Status - DescribeTable(meta::TableSchema &table_schema) override; + Status DescribeTable(meta::TableSchema &table_schema) override; - Status - HasTable(const std::string &table_id, bool &has_or_not) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; - Status - AllTables(std::vector &table_schema_array) override; + Status AllTables(std::vector &table_schema_array) override; - Status - PreloadTable(const std::string &table_id) override; + Status PreloadTable(const std::string &table_id) override; - Status - GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag); - Status - InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; - Status - Query(const std::string &table_id, + Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + + Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, QueryResults &results) override; - Status - Query(const std::string &table_id, + Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, @@ -77,8 +69,7 @@ class DBImpl : public DB { const meta::DatesT &dates, QueryResults &results) override; - Status - Query(const std::string &table_id, + Status Query(const std::string &table_id, const std::vector &file_ids, uint64_t k, uint64_t nq, diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 8dd12b0cddfca20f1dcd80bd6768abbbf88d6fd9..e227623e231bf0ff986e8d696a75ab907d824fa1 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -153,6 +153,10 @@ bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { && index1.metric_type_ == index2.metric_type_; } +bool UserDefinedId(int64_t flag) { + return flag & meta::FLAG_MASK_USERID; +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 101d849ca3b16ebf4c0fce61e88dc3d13fd4bb28..d6244ebc916b3859aed9c8a18d5c745bed652eaf 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -27,6 +27,8 @@ Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool UserDefinedId(int64_t flag); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 80ae0fb22ec2adef9b4ccb09637c362e056b55bd..93ecb588078ac97a0d621ec59b428b58b60cc0ec 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -42,6 +42,9 @@ class Meta { virtual Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) = 0; + virtual Status + UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status DeleteTable(const std::string &table_id) = 0; diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index b0c3376593273de19632a46c6c325dbd0a8e8efb..5fa4c0bcfce506b5f76542a6fcd99b6d7d3a416b 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -22,6 +22,8 @@ constexpr int32_t DEFAULT_NLIST = 16384; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int64_t FLAG_MASK_USERID = 1; + typedef int DateT; const DateT EmptyDate = -1; typedef std::vector DatesT; @@ -37,6 +39,7 @@ struct TableSchema { int32_t state_ = (int)NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; + int64_t flag_ = 0; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 954c498f7f8e9a12492dca1d2f03b950e42bc7ce..3f85b3503f4600db2c32f27011dbc555fcd8efd3 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -155,6 +155,7 @@ Status MySQLMetaImpl::Initialize() { "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << "created_on BIGINT NOT NULL, " << + "flag BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "nlist INT DEFAULT 16384 NOT NULL, " << "index_file_size INT DEFAULT 1024 NOT NULL, " << @@ -425,7 +426,7 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T "engine_type_ = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << "index_file_size = " << index.index_file_size_*ONE_MB << ", " << - "metric_type = " << index.metric_type_ << ", " << + "metric_type = " << index.metric_type_ << " " << "WHERE id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndexParam: " << updateTableIndexParamQuery.str(); @@ -455,6 +456,46 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T return Status::OK(); } +Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + try { + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + Query updateTableFlagQuery = connectionPtr->query(); + updateTableFlagQuery << "UPDATE Tables " << + "SET flag = " << flag << " " << + "WHERE id = " << quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); + + + if (!updateTableFlagQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", + updateTableFlagQuery.error()); + } + + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FLAG", er.what()); + } + + return Status::OK(); +} + Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { try { MetricCollector metric; diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 3fdd80beed37b7a7733fae214fc13f51918bcbe9..fa9880942a6dfc593c122484408956954a46a6e2 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -26,14 +26,19 @@ class MySQLMetaImpl : public Meta { MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); Status CreateTable(TableSchema &table_schema) override; + Status DescribeTable(TableSchema &group_info_) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status AllTables(std::vector &table_schema_array) override; Status DeleteTable(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; Status CreateTableFile(TableFileSchema &file_schema) override; + Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; @@ -45,6 +50,8 @@ class MySQLMetaImpl : public Meta { Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag); + Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; Status DropTableIndex(const std::string &table_id) override; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index b4859473ef306bb55cd960367271c703521c1ee4..0132fa87ceb73e471d5a8a7da178cb9b8a05d439 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -62,6 +62,7 @@ inline auto StoragePrototype(const std::string &path) { make_column("state", &TableSchema::state_), make_column("dimension", &TableSchema::dimension_), make_column("created_on", &TableSchema::created_on_), + make_column("flag", &TableSchema::flag_, default_value(0)), make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), make_column("index_file_size", &TableSchema::index_file_size_), @@ -267,6 +268,7 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, + &TableSchema::flag_, &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::index_file_size_, @@ -279,10 +281,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.state_ = std::get<1>(groups[0]); table_schema.dimension_ = std::get<2>(groups[0]); table_schema.created_on_ = std::get<3>(groups[0]); - table_schema.engine_type_ = std::get<4>(groups[0]); - table_schema.nlist_ = std::get<5>(groups[0]); - table_schema.index_file_size_ = std::get<6>(groups[0]); - table_schema.metric_type_ = std::get<7>(groups[0]); + table_schema.flag_ = std::get<4>(groups[0]); + table_schema.engine_type_ = std::get<5>(groups[0]); + table_schema.nlist_ = std::get<6>(groups[0]); + table_schema.index_file_size_ = std::get<7>(groups[0]); + table_schema.metric_type_ = std::get<8>(groups[0]); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); } @@ -358,7 +361,8 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const auto tables = ConnectorPtr->select(columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, - &TableSchema::created_on_), + &TableSchema::created_on_, + &TableSchema::flag_), where(c(&TableSchema::table_id_) == table_id and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); @@ -369,6 +373,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const table_schema.state_ = std::get<1>(tables[0]); table_schema.dimension_ = std::get<2>(tables[0]); table_schema.created_on_ = std::get<3>(tables[0]); + table_schema.flag_ = std::get<4>(tables[0]); table_schema.engine_type_ = index.engine_type_; table_schema.nlist_ = index.nlist_; table_schema.index_file_size_ = index.index_file_size_*ONE_MB; @@ -394,6 +399,28 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const std::string msg = "Encounter exception when update table index: table_id = " + table_id; return HandleException(msg, e); } + + return Status::OK(); +} + +Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + try { + MetricCollector metric; + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::flag_) = flag + ), + where( + c(&TableSchema::table_id_) == table_id + )); + + } catch (std::exception &e) { + std::string msg = "Encounter exception when update table flag: table_id = " + table_id; + return HandleException(msg, e); + } + return Status::OK(); } @@ -489,6 +516,7 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { &TableSchema::table_id_, &TableSchema::dimension_, &TableSchema::created_on_, + &TableSchema::flag_, &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::index_file_size_, @@ -498,12 +526,13 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { TableSchema schema; schema.id_ = std::get<0>(table); schema.table_id_ = std::get<1>(table); - schema.created_on_ = std::get<2>(table); - schema.dimension_ = std::get<3>(table); - schema.engine_type_ = std::get<4>(table); - schema.nlist_ = std::get<5>(table); - schema.index_file_size_ = std::get<6>(table); - schema.metric_type_ = std::get<7>(table); + schema.dimension_ = std::get<2>(table); + schema.created_on_ = std::get<3>(table); + schema.flag_ = std::get<4>(table); + schema.engine_type_ = std::get<5>(table); + schema.nlist_ = std::get<6>(table); + schema.index_file_size_ = std::get<7>(table); + schema.metric_type_ = std::get<8>(table); table_schema_array.emplace_back(schema); } diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 34808f202f7619daf9d1916d8852b3e08e2810ab..c11dce73cce51e0face4982b862dd1d6555093cf 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,82 +21,64 @@ class SqliteMetaImpl : public Meta { public: explicit SqliteMetaImpl(const DBMetaOptions &options_); - Status - CreateTable(TableSchema &table_schema) override; + Status CreateTable(TableSchema &table_schema) override; - Status - DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &group_info_) override; - Status - HasTable(const std::string &table_id, bool &has_or_not) override; + Status HasTable(const std::string &table_id, bool &has_or_not) override; - Status - AllTables(std::vector &table_schema_array) override; + Status AllTables(std::vector &table_schema_array) override; - Status - DeleteTable(const std::string &table_id) override; + Status DeleteTable(const std::string &table_id) override; - Status - DeleteTableFiles(const std::string &table_id) override; + Status DeleteTableFiles(const std::string &table_id) override; - Status - CreateTableFile(TableFileSchema &file_schema) override; + Status CreateTableFile(TableFileSchema &file_schema) override; - Status - DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; + Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; - Status - GetTableFiles(const std::string &table_id, const std::vector &ids, TableFilesSchema &table_files) override; + Status GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) override; + + Status HasNonIndexFiles(const std::string &table_id, bool &has) override; - Status - HasNonIndexFiles(const std::string &table_id, bool &has) override; + Status UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; - Status - UpdateTableIndexParam(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status - DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; - Status - DropTableIndex(const std::string &table_id) override; + Status DropTableIndex(const std::string &table_id) override; - Status - UpdateTableFilesToIndex(const std::string &table_id) override; + Status UpdateTableFilesToIndex(const std::string &table_id) override; - Status - UpdateTableFile(TableFileSchema &file_schema) override; + Status UpdateTableFile(TableFileSchema &file_schema) override; - Status - UpdateTableFiles(TableFilesSchema &files) override; + Status UpdateTableFiles(TableFilesSchema &files) override; - Status - FilesToSearch(const std::string &table_id, const DatesT &partition, DatePartionedTableFilesSchema &files) override; + Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) override; Status FilesToSearch(const std::string &table_id, const std::vector &ids, const DatesT &partition, DatePartionedTableFilesSchema &files) override; - Status - FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; + Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; - Status - FilesToIndex(TableFilesSchema &) override; + Status FilesToIndex(TableFilesSchema &) override; - Status - Archive() override; + Status Archive() override; - Status - Size(uint64_t &result) override; + Status Size(uint64_t &result) override; - Status - CleanUp() override; + Status CleanUp() override; - Status - CleanUpFilesWithTTL(uint16_t seconds) override; + Status CleanUpFilesWithTTL(uint16_t seconds) override; - Status - DropAll() override; + Status DropAll() override; Status Count(const std::string &table_id, uint64_t &result) override; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 4e7c0f461188945515c7e56d1d692ead3abeee1e..70802e386d1bd523933846a7760058a640261ba4 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -16,7 +16,7 @@ namespace engine { namespace { -static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 10000; +static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 1000000; static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; bool NeedParallelReduce(uint64_t nq, uint64_t topk) { diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index f30a23e174d3568797453f3f65602fd56d73a23c..3f43a9619c5b0d5d2162de64d612194ab516de83 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -236,7 +236,6 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::vector> search_record_array; {//insert vectors - std::vector record_ids; for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; @@ -249,6 +248,12 @@ ClientTest::Test(const std::string& address, const std::string& port) { } #endif + std::vector record_ids; + //generate user defined ids + for(int k = 0; k < BATCH_ROW_COUNT; k++) { + record_ids.push_back(i*BATCH_ROW_COUNT+k); + } + auto start = std::chrono::high_resolution_clock::now(); Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index a84e9a2dd83184045673d0d39f999fcd0e0f7923..923f8f2861d51c251e628e64f0968f7b42acf296 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -74,12 +74,6 @@ DBWrapper::DBWrapper() { } } - std::string metric_type = engine_config.GetValue(CONFIG_METRICTYPE, "L2"); - if(metric_type != "L2" && metric_type != "IP") { - std::cout << "ERROR! Illegal metric type: " << metric_type << ", available options: L2 or IP" << std::endl; - kill(0, SIGUSR1); - } - //set archive config engine::ArchiveConf::CriteriaT criterial; int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 6a76399d42c92e856f353ddf4cb04e9cd8414cc9..a800664ebd38b83f38a02a8011113db35e306ce3 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -17,7 +17,6 @@ namespace server { static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const char* CONFIG_CLUSTER_MODE = "mode"; static const char* CONFIG_GPU_INDEX = "gpu_index"; @@ -41,9 +40,6 @@ static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately"; static const char* CONFIG_GPU_IDS = "gpu_ids"; static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent"; -static const char* CONFIG_LICENSE = "license_config"; -static const char* CONFIG_LICENSE_PATH = "license_path"; - static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC_IS_STARTUP = "is_startup"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; @@ -51,13 +47,8 @@ static const char* CONFIG_PROMETHEUS = "prometheus_config"; static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; static const std::string CONFIG_ENGINE = "engine_config"; -static const std::string CONFIG_NPROBE = "nprobe"; -static const std::string CONFIG_NLIST = "nlist"; static const std::string CONFIG_DCBT = "use_blas_threshold"; -static const std::string CONFIG_METRICTYPE = "metric_type"; static const std::string CONFIG_OMP_THREAD_NUM = "omp_thread_num"; -static const std::string CONFIG_USE_HYBRID_INDEX = "use_hybrid_index"; -static const std::string CONFIG_HYBRID_INDEX_GPU = "hybrid_index_gpu"; class ServerConfig { public: diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 95ceaffd42e8396961b639e4af533e307a028692..31c4ebf5b158f70fd03a0e50645c48591512cb6a 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -12,9 +12,12 @@ #include "../DBWrapper.h" #include "version.h" #include "GrpcMilvusServer.h" +#include "db/Utils.h" #include "src/server/Server.h" +#include + namespace zilliz { namespace milvus { namespace server { @@ -435,6 +438,23 @@ InsertTask::OnExecute() { } } + //all user provide id, or all internal id + uint64_t row_count = 0; + DBWrapper::DB()->GetTableRowCount(table_info.table_id_, row_count); + bool empty_table = (row_count == 0); + bool user_provide_ids = !insert_param_.row_id_array().empty(); + if(!empty_table) { + //user already provided id before, all insert action require user id + if(engine::utils::UserDefinedId(table_info.flag_) && !user_provide_ids) { + return SetError(SERVER_INVALID_ARGUMENT, "Table vector ids are user defined, please provide id for this batch"); + } + + //user didn't provided id before, no need to provide user id + if(!engine::utils::UserDefinedId(table_info.flag_) && user_provide_ids) { + return SetError(SERVER_INVALID_ARGUMENT, "Table vector ids are auto generated, no need to provide id for this batch"); + } + } + rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING @@ -469,8 +489,10 @@ InsertTask::OnExecute() { //step 4: insert vectors auto vec_count = (uint64_t) insert_param_.row_record_array_size(); std::vector vec_ids(insert_param_.row_id_array_size(), 0); - for (auto i = 0; i < insert_param_.row_id_array_size(); i++) { - vec_ids[i] = insert_param_.row_id_array(i); + if(!insert_param_.row_id_array().empty()) { + const int64_t* src_data = insert_param_.row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, (size_t)(sizeof(int64_t)*insert_param_.row_id_array_size())); } stat = DBWrapper::DB()->InsertVectors(insert_param_.table_name(), vec_count, vec_f.data(), vec_ids); @@ -489,6 +511,12 @@ InsertTask::OnExecute() { return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } + //step 5: update table flag + if(empty_table && user_provide_ids) { + stat = DBWrapper::DB()->UpdateTableFlag(insert_param_.table_name(), + table_info.flag_ | engine::meta::FLAG_MASK_USERID); + } + #ifdef MILVUS_ENABLE_PROFILING ProfilerStop(); #endif diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index 1997748fddf3aa6e4618de8b9b1567fe09045976..feda6d1bf9aea3ccabe2f07264dcf05fc53681d6 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -21,7 +21,8 @@ TEST(MetricbaseTest, METRICBASE_TEST){ instance.RawFileSizeHistogramObserve(1.0); instance.IndexFileSizeHistogramObserve(1.0); instance.BuildIndexDurationSecondsHistogramObserve(1.0); - instance.CacheUsageGaugeSet(1.0); + instance.CpuCacheUsageGaugeSet(1.0); + instance.GpuCacheUsageGaugeSet(1.0); instance.MetaAccessTotalIncrement(); instance.MetaAccessDurationSecondsHistogramObserve(1.0); instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 004e58a5fc5ff26e66fe9699c453062355b0b8e1..ece3cf9012b0093f1a5f71f9393011a88cfbfddd 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -22,7 +22,8 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.RawFileSizeHistogramObserve(1.0); instance.IndexFileSizeHistogramObserve(1.0); instance.BuildIndexDurationSecondsHistogramObserve(1.0); - instance.CacheUsageGaugeSet(1.0); + instance.CpuCacheUsageGaugeSet(1.0); + instance.GpuCacheUsageGaugeSet(1.0); instance.MetaAccessTotalIncrement(); instance.MetaAccessDurationSecondsHistogramObserve(1.0); instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);