diff --git a/CHANGELOGS.md b/CHANGELOGS.md index d9ef3d4c36779994fdfc209d504cddec0d9a81af..bbe0d9fc6077566b163422e7ccdbc9cb3150fd39 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -17,3 +17,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-1 - Add CHANGELOG.md - MS-161 - Add CI / CD Module to Milvus Project - MS-202 - Add Milvus Jenkins project email notification +- MS-215 - Add Milvus cluster CI/CD groovy file diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 946e08e99cc275fa02ae4390846b3937ec122f91..157103df22432acbd9016b41af6d7e8dea544be3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-153 - fix c_str error when connecting to MySQL - MS-157 - fix changelog - MS-190 - use env variable to switch mem manager and fix cmake +- MS-217 - Fix SQ8 row count bug - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists ## Improvement diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 17d5bcadefab84b243d848476a9e6d27e1fcc344..43b6f3ba1661dfa1d88c7b96d90e2dabe90a3e71 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -14,8 +14,8 @@ db_config: db_backend_url: sqlite://:@:/ index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB - archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB - archive_days_threshold: 30 # files older than x days will be archived, unit: day + archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB + archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB. # the sum of maximum_memory and cpu_cache_capacity should be less than total memory @@ -39,4 +39,4 @@ engine_config: nprobe: 10 nlist: 16384 use_blas_threshold: 20 - metric_type: L2 #L2 or Inner Product + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 1dff04027eb5b1b0574b4818b484fb7a48e731ea..995711c6ab7e0ee72cf37497270952a2ec588f8a 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -20,6 +20,11 @@ public: : index_(index) {} + DataObj(const engine::Index_ptr& index, int64_t size) + : index_(index), + size_(size) + {} + engine::Index_ptr data() { return index_; } const engine::Index_ptr& data() const { return index_; } @@ -28,11 +33,16 @@ public: return 0; } + if(size_ > 0) { + return size_; + } + return index_->ntotal*(index_->dim*4); } private: engine::Index_ptr index_ = nullptr; + int64_t size_ = 0; }; using DataObjPtr = std::shared_ptr; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index dca0f2a2ed0201102ab800dd026aec795fa6de6d..55fc9cea5aeb414c1b9710febfe56a32f1076da0 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -169,7 +169,10 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, } } - return QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, @@ -194,7 +197,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return Status::Error("Invalid file id"); } - return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, @@ -486,7 +492,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->PhysicalSize(); + table_file.size_ = index->Size(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index fc67c6b36029248f9a279ec8fcf97a85bde745a9..99ae3b6711a7aa8edfdcac8dfb0657f829d83d7c 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -674,16 +674,22 @@ Status DBMetaImpl::Archive() { Status DBMetaImpl::Size(uint64_t &result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::file_type_, + &TableFileSchema::engine_type_), + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE + )); - for (auto &sub_query : selected) { - if (!std::get<0>(sub_query)) { - continue; + for (auto &file : files) { + auto file_size = std::get<0>(file); + auto file_type = std::get<1>(file); + auto engine_type = std::get<2>(file); + if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) { + result += (uint64_t)file_size/4;//hardcode for sq8 + } else { + result += (uint64_t)file_size; } - result += (uint64_t) (*std::get<0>(sub_query)); } } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index d2cb835364291b3ba62dfc1002d6978c8f684e5e..e32363dd1d208def1095d9cc4ce6b628485c4668 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -110,7 +110,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; + ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_; auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); if (!to_merge) { @@ -165,8 +165,9 @@ Status FaissExecutionEngine::Search(long n, } Status FaissExecutionEngine::Cache() { - zilliz::milvus::cache::CpuCacheMgr::GetInstance( - )->InsertItem(location_, std::make_shared(pIndex_)); + auto index = std::make_shared(pIndex_); + cache::DataObjPtr data_obj = std::make_shared(index, PhysicalSize()); + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, data_obj); return Status::OK(); } diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5f591dcb0f2536423294728770af71ee83d44f61..81f97978eaaaef515f2fb98aa6111de98a139a2c 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -41,6 +41,10 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { } for (auto& token : tokens) { + if(token.empty()) { + continue; + } + std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 515a84d6634d2adcbe237c8f72e35589effed23d..b8e049f9d4fb01711e890f02d67d06e1c6362622 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -22,7 +22,7 @@ static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; static const std::string ARCHIVE_CONF_DISK = "disk"; static const std::string ARCHIVE_CONF_DAYS = "days"; -static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; +static const std::string ARCHIVE_CONF_DEFAULT = ""; struct ArchiveConf { using CriteriaT = std::map; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 8036cf986dba1a080fb840fdd4b2a7f139f8454b..5606ba2c84221712e648268d53325f96bf3b58ff 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -107,7 +107,7 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, uint64_t nq, uint64_t topk, SearchContext::ResultSet &result_set) { - if(output_ids.size() != nq*topk || output_distence.size() != nq*topk) { + if(output_ids.size() < nq*topk || output_distence.size() < nq*topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); SERVER_LOG_ERROR << msg; diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 495e6ae86133746afdadf183beb3f85363b799d4..5caab0961dd1eb4cff8344e0a195259f08bf8de4 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -23,6 +23,7 @@ namespace { static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different static constexpr int64_t ADD_VECTOR_LOOP = 10; + static constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -59,7 +60,7 @@ namespace { std::string CurrentTime() { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) @@ -69,10 +70,11 @@ namespace { return str; } - std::string CurrentTmDate() { + std::string CurrentTmDate(int64_t offset_day = 0) { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; + tt = tt + 24*SECONDS_EACH_HOUR*offset_day; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) @@ -160,7 +162,7 @@ namespace { std::vector query_range_array; Range rg; rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(1); query_range_array.emplace_back(rg); std::vector record_array; diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 62a863ae0eeb027ce07ce26ce3a87e062701fea9..49d007ba2af897d3cc306247f3f4f6589b7866fa 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -28,6 +28,15 @@ IndexType resolveIndexType(const string &index_type) { return IndexType::Invalid_Option; } +int CalcBacketCount(int nb, size_t nlist) { + int backet_count = int(nb / 1000000.0 * nlist); + if(backet_count == 0) { + backet_count = 1; //avoid faiss rash + } + + return backet_count; +} + // nb at least 100 string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } @@ -45,7 +54,7 @@ string Operand::get_index_type(const int &nb) { size_t nlist = engine_config.GetInt32Value(CONFIG_NLIST, 16384); index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * nlist))); + index_type + std::to_string(CalcBacketCount(nb, nlist))); // std::cout<<"nlist = "<BuildIndex(TABLE_NAME); // wait until build index finish - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); + { + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + } + + {//search by specify index file + engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4"}; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, file_ids, k, nq, xq.data(), dates, results); + ASSERT_STATS(stat); + } // TODO(linxj): add groundTruth assert }; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 4356746fc2a42a325371ea639bbda39622486ea8..9dd07fac8d89b55e8db1099c08bcbe0ff099802b 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -14,6 +14,7 @@ #include "db/Options.h" #include "db/DBMetaImpl.h" #include "db/EngineFactory.h" +#include "db/Utils.h" #include @@ -134,4 +135,32 @@ TEST(DBMiscTest, META_TEST) { int delta = 10; engine::meta::DateT dt = impl.GetDate(tt, delta); ASSERT_GT(dt, 0); +} + +TEST(DBMiscTest, UTILS_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test/main"; + options.slave_paths.push_back("/tmp/milvus_test/slave_1"); + options.slave_paths.push_back("/tmp/milvus_test/slave_2"); + + const std::string TABLE_NAME = "test_tbl"; + auto status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(boost::filesystem::exists(options.path)); + for(auto& path : options.slave_paths) { + ASSERT_TRUE(boost::filesystem::exists(path)); + } + + engine::meta::TableFileSchema file; + file.id_ = 50; + file.table_id_ = TABLE_NAME; + file.file_type_ = 3; + file.date_ = 155000; + status = engine::utils::GetTableFilePath(options, file); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(file.location_.empty()); + + status = engine::utils::DeleteTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + } \ No newline at end of file