Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 1040298e 编写于 作者: S starlord
浏览文件
操作

refine code 


Former-commit-id: 5303054b0769e7f03a16ede2ff42f7068663b889
上级 e62a7b36
隐藏空白更改
内联 并排
Showing with 49 addition and 58 deletion
cpp/src/db/meta/MetaConsts.h
浏览文件 @ 1040298e
......@@ -21,11 +21,6 @@ namespace milvus {
namespace engine {
namespace meta {
const size_t K = 1024UL;
const size_t M = K * K;
const size_t G = K * M;
const size_t T = K * G;
const size_t S_PS = 1UL;
const size_t MS_PS = 1000 * S_PS;
const size_t US_PS = 1000 * MS_PS;
......
cpp/src/scheduler/ResourceMgr.cpp
浏览文件 @ 1040298e
......@@ -79,9 +79,7 @@ ResourceMgr::Add(ResourcePtr&& resource) {
gpu_resources_.emplace_back(ResourceWPtr(resource));
break;
}
default: {
break;
}
default: { break; }
}
resources_.emplace_back(resource);
......@@ -196,13 +194,19 @@ bool
ResourceMgr::check_resource_valid() {
{
// TODO: check one disk-resource, one cpu-resource, zero or more gpu-resource;
if (GetDiskResources().size() != 1) return false;
if (GetCpuResources().size() != 1) return false;
if (GetDiskResources().size() != 1) {
return false;
}
if (GetCpuResources().size() != 1) {
return false;
}
}
{
// TODO: one compute-resource at least;
if (GetNumOfComputeResource() < 1) return false;
if (GetNumOfComputeResource() < 1) {
return false;
}
}
{
......@@ -233,9 +237,7 @@ void
ResourceMgr::event_process() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] {
return !queue_.empty();
});
event_cv_.wait(lock, [this] { return !queue_.empty(); });
auto event = queue_.front();
queue_.pop();
......
cpp/src/scheduler/task/SearchTask.cpp
浏览文件 @ 1040298e
......@@ -22,6 +22,7 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <algorithm>
#include <string>
#include <thread>
#include <utility>
......@@ -229,13 +230,8 @@ XSearchTask::Execute() {
}
Status
XSearchTask::TopkResult(const std::vector<long> &input_ids,
const std::vector<float> &input_distance,
uint64_t input_k,
uint64_t nq,
uint64_t topk,
bool ascending,
scheduler::ResultSet &result) {
XSearchTask::TopkResult(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance,
uint64_t input_k, uint64_t nq, uint64_t topk, bool ascending, scheduler::ResultSet& result) {
scheduler::ResultSet result_buf;
if (result.empty()) {
......@@ -265,7 +261,7 @@ XSearchTask::TopkResult(const std::vector<long> &input_ids,
auto& result_buf_item = result_buf_i[buf_k];
auto& result_item = result_i[tar_k];
if ((ascending && input_distance[src_idx] < result_item.second) ||
(!ascending && input_distance[src_idx] > result_item.second)) {
(!ascending && input_distance[src_idx] > result_item.second)) {
result_buf_item.first = input_ids[src_idx];
result_buf_item.second = input_distance[src_idx];
src_k++;
......
cpp/src/scheduler/task/SearchTask.h
浏览文件 @ 1040298e
......@@ -39,13 +39,8 @@ class XSearchTask : public Task {
public:
static Status
TopkResult(const std::vector<long> &input_ids,
const std::vector<float> &input_distance,
uint64_t input_k,
uint64_t nq,
uint64_t topk,
bool ascending,
scheduler::ResultSet &result);
TopkResult(const std::vector<int64_t>& input_ids, const std::vector<float>& input_distance, uint64_t input_k,
uint64_t nq, uint64_t topk, bool ascending, scheduler::ResultSet& result);
public:
TableFileSchemaPtr file_;
......
cpp/unittest/db/test_db.cpp
浏览文件 @ 1040298e
......@@ -18,6 +18,7 @@
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include "db/DBFactory.h"
#include "cache/CpuCacheMgr.h"
......@@ -190,7 +191,7 @@ TEST_F(DBTest, DB_TEST) {
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
......@@ -439,7 +440,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * ms::engine::meta::G);
ASSERT_LE(size, 1 * ms::engine::G);
}
TEST_F(DBTest2, DELETE_TEST) {
......
cpp/unittest/db/test_db_mysql.cpp
浏览文件 @ 1040298e
......@@ -18,6 +18,7 @@
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include <gtest/gtest.h>
......@@ -98,7 +99,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
......@@ -236,7 +237,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * ms::engine::meta::G);
ASSERT_LE(size, 1 * ms::engine::G);
}
TEST_F(MySqlDBTest, DELETE_TEST) {
......
cpp/unittest/db/test_mem.cpp
浏览文件 @ 1040298e
......@@ -329,7 +329,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
START_TIMER;
stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
......
cpp/unittest/db/test_meta.cpp
浏览文件 @ 1040298e
......@@ -18,6 +18,7 @@
#include "db/utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/Constants.h"
#include "db/meta/MetaConsts.h"
#include <gtest/gtest.h>
......@@ -192,7 +193,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
table_file.file_size_ = each_size * ms::engine::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
......
cpp/unittest/db/test_meta_mysql.cpp
浏览文件 @ 1040298e
......@@ -213,7 +213,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
table_file.file_size_ = each_size * ms::engine::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
......
cpp/unittest/db/test_search.cpp
浏览文件 @ 1040298e
......@@ -22,10 +22,10 @@
#include "scheduler/task/SearchTask.h"
#include "utils/TimeRecorder.h"
using namespace milvus::scheduler;
namespace {
namespace ms = milvus::scheduler;
void
BuildResult(uint64_t nq,
uint64_t topk,
......@@ -45,14 +45,14 @@ BuildResult(uint64_t nq,
}
}
void CheckTopkResult(const std::vector<long> &input_ids_1,
void CheckTopkResult(const std::vector<int64_t> &input_ids_1,
const std::vector<float> &input_distance_1,
const std::vector<long> &input_ids_2,
const std::vector<int64_t> &input_ids_2,
const std::vector<float> &input_distance_2,
uint64_t nq,
uint64_t topk,
bool ascending,
const ResultSet& result) {
const ms::ResultSet& result) {
ASSERT_EQ(result.size(), nq);
ASSERT_EQ(input_ids_1.size(), input_distance_1.size());
ASSERT_EQ(input_ids_2.size(), input_distance_2.size());
......@@ -85,21 +85,21 @@ TEST(DBSearchTest, TOPK_TEST) {
uint64_t NQ = 15;
uint64_t TOP_K = 64;
bool ascending;
std::vector<long> ids1, ids2;
std::vector<int64_t> ids1, ids2;
std::vector<float> dist1, dist2;
ResultSet result;
ms::ResultSet result;
milvus::Status status;
/* test1, id1/dist1 valid, id2/dist2 empty */
ascending = true;
BuildResult(NQ, TOP_K, ascending, ids1, dist1);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
/* test2, id1/dist1 valid, id2/dist2 valid */
BuildResult(NQ, TOP_K, ascending, ids2, dist2);
status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
......@@ -108,9 +108,9 @@ TEST(DBSearchTest, TOPK_TEST) {
dist1.clear();
result.clear();
BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
......@@ -119,9 +119,9 @@ TEST(DBSearchTest, TOPK_TEST) {
dist2.clear();
result.clear();
BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
......@@ -135,13 +135,13 @@ TEST(DBSearchTest, TOPK_TEST) {
/* test1, id1/dist1 valid, id2/dist2 empty */
BuildResult(NQ, TOP_K, ascending, ids1, dist1);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
/* test2, id1/dist1 valid, id2/dist2 valid */
BuildResult(NQ, TOP_K, ascending, ids2, dist2);
status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
......@@ -150,9 +150,9 @@ TEST(DBSearchTest, TOPK_TEST) {
dist1.clear();
result.clear();
BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
......@@ -161,9 +161,9 @@ TEST(DBSearchTest, TOPK_TEST) {
dist2.clear();
result.clear();
BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
status = XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
}
......@@ -173,9 +173,9 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
int32_t top_k = 1000;
int32_t index_file_num = 478; /* sift1B dataset, index files num */
bool ascending = true;
std::vector<long> input_ids;
std::vector<int64_t> input_ids;
std::vector<float> input_distance;
ResultSet final_result;
ms::ResultSet final_result;
milvus::Status status;
double span, reduce_cost = 0.0;
......@@ -187,7 +187,7 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
rc.RecordSection("do search for context: " + std::to_string(i));
// pick up topk result
status = XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result);
status = ms::XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(final_result.size(), nq);
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或