Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 7856fdd9 编写于 作者: P peng.xu
浏览文件
操作

Merge branch 'branch-0.4.0' into 'branch-0.4.0' 

fix row count bug

See merge request megasearch/milvus!403

Former-commit-id: 295cf00af668f8af89d4c3bfcbbb2b58a89b81a8
上级 59ff9b57 f0d9b0f5
隐藏空白更改
内联 并排
Showing with 68 addition and 30 deletion
cpp/src/db/insert/MemTableFile.cpp
浏览文件 @ 7856fdd9
......@@ -86,6 +86,8 @@ Status MemTableFile::Serialize() {
execution_engine_->Serialize();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
table_file_schema_.row_count_ = execution_engine_->Count();
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size / total_time);
......
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 7856fdd9
......@@ -652,7 +652,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) {
}
Query describeTableQuery = connectionPtr->query();
describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " <<
describeTableQuery << "SELECT id, dimension, engine_type " <<
"FROM Tables " <<
"WHERE table_id = " << quote << table_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
......@@ -739,7 +739,7 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
}
Query allTablesQuery = connectionPtr->query();
allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " <<
allTablesQuery << "SELECT id, table_id, dimension, engine_type " <<
"FROM Tables " <<
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
......@@ -864,7 +864,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
}
Query filesToIndexQuery = connectionPtr->query();
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " <<
filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on " <<
"FROM TableFiles " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";";
......@@ -891,10 +891,14 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
table_file.file_type_ = resRow["file_type"];
table_file.file_size_ = resRow["file_size"];
table_file.row_count_ = resRow["row_count"];
table_file.date_ = resRow["date"];
table_file.created_on_ = resRow["created_on"];
auto groupItr = groups.find(table_file.table_id_);
if (groupItr == groups.end()) {
TableSchema table_schema;
......@@ -943,7 +947,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
if (partition.empty()) {
Query filesToSearchQuery = connectionPtr->query();
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " <<
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " <<
......@@ -965,7 +969,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
std::string partitionListStr = partitionListSS.str();
partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", "
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " <<
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"date IN (" << partitionListStr << ") AND " <<
......@@ -1004,6 +1008,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
table_file.file_type_ = resRow["file_type"];
table_file.file_size_ = resRow["file_size"];
table_file.row_count_ = resRow["row_count"];
table_file.date_ = resRow["date"];
......@@ -1049,7 +1055,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
}
Query filesToSearchQuery = connectionPtr->query();
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, row_count, date " <<
filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id;
......@@ -1110,6 +1116,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
table_file.file_type_ = resRow["file_type"];
table_file.file_size_ = resRow["file_size"];
table_file.row_count_ = resRow["row_count"];
table_file.date_ = resRow["date"];
......@@ -1153,7 +1161,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
}
Query filesToMergeQuery = connectionPtr->query();
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, date " <<
filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, create_on " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type = " << std::to_string(TableFileSchema::RAW) << " " <<
......@@ -1189,8 +1197,14 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
table_file.file_size_ = resRow["file_size"];
table_file.row_count_ = resRow["row_count"];
table_file.date_ = resRow["date"];
table_file.engine_type_ = resRow["engine_type"];
table_file.created_on_ = resRow["created_on"];
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
......@@ -1241,7 +1255,7 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
Query getTableFileQuery = connectionPtr->query();
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date " <<
getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " <<
"FROM TableFiles " <<
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ");";
......@@ -1280,6 +1294,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
file_schema.date_ = resRow["date"];
file_schema.created_on_ = resRow["created_on"];
file_schema.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, file_schema);
......
cpp/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 7856fdd9
......@@ -551,9 +551,11 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::file_size_,
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
&TableFileSchema::engine_type_,
&TableFileSchema::created_on_),
where(c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX));
......@@ -565,9 +567,11 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.row_count_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.file_size_ = std::get<4>(file);
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.created_on_ = std::get<8>(file);
utils::GetTableFilePath(options_, table_file);
auto groupItr = groups.find(table_file.table_id_);
......@@ -605,6 +609,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::file_size_,
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
......@@ -625,9 +630,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.row_count_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.file_size_ = std::get<4>(file);
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
......@@ -643,6 +649,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::file_size_,
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
......@@ -664,9 +671,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.row_count_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.file_size_ = std::get<4>(file);
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
......@@ -696,6 +704,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::file_size_,
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_);
......@@ -738,9 +747,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.row_count_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.file_size_ = std::get<4>(file);
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
......@@ -769,7 +779,9 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::file_size_,
&TableFileSchema::date_),
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::created_on_),
where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and
c(&TableFileSchema::table_id_) == table_id),
order_by(&TableFileSchema::file_size_).desc());
......@@ -789,7 +801,9 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.file_size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.created_on_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
......@@ -816,7 +830,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
&TableFileSchema::file_size_,
&TableFileSchema::row_count_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
&TableFileSchema::engine_type_,
&TableFileSchema::created_on_),
where(c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::id_, ids)
));
......@@ -838,6 +853,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
file_schema.row_count_ = std::get<4>(file);
file_schema.date_ = std::get<5>(file);
file_schema.engine_type_ = std::get<6>(file);
file_schema.created_on_ = std::get<7>(file);
file_schema.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, file_schema);
......@@ -1218,9 +1234,6 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
result += std::get<0>(file);
}
result /= table_schema.dimension_;
result /= sizeof(float);
} catch (std::exception &e) {
return HandleException("Encounter exception when calculate table file size", e);
}
......
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
浏览文件 @ 7856fdd9
......@@ -86,9 +86,8 @@ namespace {
}
std::string GetTableName() {
// static std::string s_id(CurrentTime());
// return "tbl_" + s_id;
return "test";
static std::string s_id(CurrentTime());
return "tbl_" + s_id;
}
TableSchema BuildTableSchema() {
......@@ -269,6 +268,10 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//search vectors without index
Sleep(2);
int64_t row_count = 0;
Status stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
DoSearch(conn, search_record_array, "Search without index");
}
......@@ -300,6 +303,10 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//delete index
Status stat = conn->DropIndex(TABLE_NAME);
std::cout << "DropIndex function call status: " << stat.ToString() << std::endl;
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{//delete by range
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或