diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 9ac6cb11bce700db4e4b8f5be796e2a2f4702832..3be585c7b6f2dc641677e85b6c4b77dae7ceec5c 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,8 +9,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog +- MS-153 - Fix c_str error when connecting to MySQL +- MS-157 - Fix changelog +- MS-217 - Fix SQ8 row count bug ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3f3b004456b56513e46bc4633aae7bc41a71e6a6..6de96ae89585a49fece1dce12f4286367a0a5030 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -14,8 +14,8 @@ db_config: db_backend_url: sqlite://:@:/ index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB - archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB - archive_days_threshold: 30 # files older than x days will be archived, unit: day + archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB + archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day metric_config: is_startup: off # if monitoring start: on, off @@ -37,4 +37,4 @@ engine_config: nprobe: 10 nlist: 16384 use_blas_threshold: 20 - metric_type: L2 #L2 or Inner Product \ No newline at end of file + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP \ No newline at end of file diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 35a6f075f8ec726a68aef43155cdea23090f978d..beae6c30d31cdbaa6414f00f0934a76adcf1dc4c 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->PhysicalSize(); + table_file.size_ = index->Size(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index fc67c6b36029248f9a279ec8fcf97a85bde745a9..99ae3b6711a7aa8edfdcac8dfb0657f829d83d7c 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -674,16 +674,22 @@ Status DBMetaImpl::Archive() { Status DBMetaImpl::Size(uint64_t &result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::file_type_, + &TableFileSchema::engine_type_), + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE + )); - for (auto &sub_query : selected) { - if (!std::get<0>(sub_query)) { - continue; + for (auto &file : files) { + auto file_size = std::get<0>(file); + auto file_type = std::get<1>(file); + auto engine_type = std::get<2>(file); + if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) { + result += (uint64_t)file_size/4;//hardcode for sq8 + } else { + result += (uint64_t)file_size; } - result += (uint64_t) (*std::get<0>(sub_query)); } } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index d2cb835364291b3ba62dfc1002d6978c8f684e5e..7183f430171c07b47a2062d77dd214b8c8e22375 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -110,7 +110,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; + ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_; auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); if (!to_merge) { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 1c3c28b4ac6ff2fa653a0b53b5d1aad6ae406bbf..f6c09fb75041db9f286b2fefb25fd369fbad459b 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -22,7 +22,7 @@ static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; static const std::string ARCHIVE_CONF_DISK = "disk"; static const std::string ARCHIVE_CONF_DAYS = "days"; -static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; +static const std::string ARCHIVE_CONF_DEFAULT = ""; struct ArchiveConf { using CriteriaT = std::map;