diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 3d3a67f965abaec6a7c3206df570286955c3e14e..2a0595b2cabe751cbd594d9dfec601c3568a8079 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -4,6 +4,7 @@ server_config: transfer_protocol: binary #optional: binary, compact, json server_mode: thread_pool #optional: simple, thread_pool gpu_index: 0 #which gpu to be used + mode: local #optional: local, cluster db_config: db_path: /tmp/milvus diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 7642ea37a759189b1267d567ddda145a1e196864..01ea9d29bbbcb04a382623c414d6c1490e487c37 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -227,15 +227,18 @@ Status DBImpl::Query(const std::string& table_id, const std::vector uint64_t k, uint64_t nq, const float* vectors, const meta::DatesT& dates, QueryResults& results) { //get specified files - meta::TableFilesSchema files_array; + std::vector ids; for (auto &id : file_ids) { meta::TableFileSchema table_file; - table_file.table_id_ = id; - auto status = meta_ptr_->GetTableFile(table_file); - if (!status.ok()) { - return status; - } - files_array.emplace_back(table_file); + table_file.table_id_ = table_id; + std::string::size_type sz; + ids.push_back(std::stol(id, &sz)); + } + + meta::TableFilesSchema files_array; + auto status = meta_ptr_->GetTableFiles(table_id, ids, files_array); + if (!status.ok()) { + return status; } return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index dbbc985067f60b7929cee418c9b5d716c31e689e..c6c80ab9ddea05ebc47339e425b66a17772bf5fa 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -604,32 +604,41 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id, return Status::OK(); } -Status DBMetaImpl::GetTableFile(TableFileSchema &file_schema) { - +Status DBMetaImpl::GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) { try { - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::table_id_, - &TableFileSchema::file_id_, + table_files.clear(); + auto files = ConnectorPtr->select(columns(&TableFileSchema::file_id_, &TableFileSchema::file_type_, &TableFileSchema::size_, - &TableFileSchema::date_), - where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and - c(&TableFileSchema::table_id_) == file_schema.table_id_ + &TableFileSchema::date_, + &TableFileSchema::engine_type_), + where(c(&TableFileSchema::table_id_) == table_id and + in(&TableFileSchema::id_, ids) )); - assert(files.size() <= 1); - if (files.size() == 1) { - file_schema.id_ = std::get<0>(files[0]); - file_schema.table_id_ = std::get<1>(files[0]); - file_schema.file_id_ = std::get<2>(files[0]); - file_schema.file_type_ = std::get<3>(files[0]); - file_schema.size_ = std::get<4>(files[0]); - file_schema.date_ = std::get<5>(files[0]); - } else { - return Status::NotFound("Table:" + file_schema.table_id_ + - " File:" + file_schema.file_id_ + " not found"); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + for (auto &file : files) { + TableFileSchema file_schema; + file_schema.file_id_ = std::get<0>(file); + file_schema.file_type_ = std::get<1>(file); + file_schema.size_ = std::get<2>(file); + file_schema.date_ = std::get<3>(file); + file_schema.engine_type_ = std::get<4>(file); + file_schema.dimension_ = table_schema.dimension_; + GetTableFilePath(file_schema); + + table_files.emplace_back(file_schema); } } catch (std::exception &e) { - return HandleException("Encounter exception when lookup table file", e); + return HandleException("Encounter exception when lookup table files", e); } return Status::OK(); diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 21ce8d1df6a96306155843cf84b73745c7e9b1a3..1d5f400622a395fa3635eda857580401f01e8f45 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -31,7 +31,9 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - virtual Status GetTableFile(TableFileSchema& file_schema) override; + virtual Status GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) override; virtual Status UpdateTableFile(TableFileSchema& file_schema) override; diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 3bd60cc7feaaeba5817629df48cb02ec753d1222..6c6dd28bd0db52929646bf131f210f9eb56e417e 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -35,14 +35,17 @@ public: virtual Status DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; - virtual Status GetTableFile(TableFileSchema& file_schema) = 0; + virtual Status GetTableFiles(const std::string& table_id, + const std::vector& ids, + TableFilesSchema& table_files) = 0; + virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; - virtual Status FilesToSearch(const std::string& table_id, - const DatesT& partition, - DatePartionedTableFilesSchema& files) = 0; + virtual Status FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema& files) = 0; virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 147ab3859548180848e925d742bab4c574fbb23e..d71a942d3e5c3262bf031810e7deff5b04eddfa6 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -85,9 +85,13 @@ TEST_F(MetaTest, table_file_TEST) { dates.push_back(table_file.date_); status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - status = impl_->GetTableFile(table_file); + + std::vector ids = {table_file.id_}; + meta::TableFilesSchema files; + status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(files.size(), 1UL); + ASSERT_TRUE(files[0].file_type_ == meta::TableFileSchema::TO_DELETE); } TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { @@ -113,6 +117,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); std::vector days; + std::vector ids; for (auto i=0; i ids; for (auto i=0; i