Skip to content

M
milvus

milvus / milvus
大约 1 年 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 faa2c4b4 编写于 作者: G groot
浏览文件
操作

support empty id 


Former-commit-id: 1a5303b58f3f3b0154f1609ff1acc0d00007c569
上级 496989bb
展开全部 隐藏空白更改
内联 并排
Showing with 613 addition and 261 deletion
cpp/src/server/VecServiceHandler.cpp
浏览文件 @ faa2c4b4
......@@ -74,46 +74,49 @@ VecServiceHandler::del_group(const std::string &group_id) {
void
VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tensor) {
VecServiceHandler::add_vector(std::string& _return, const std::string &group_id, const VecTensor &tensor) {
TimeRecordWrapper rc("add_vector()");
SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size();
BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor);
BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return);
VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance();
scheduler.ExecuteTask(task_ptr);
}
void
VecServiceHandler::add_vector_batch(const std::string &group_id,
VecServiceHandler::add_vector_batch(std::vector<std::string> & _return,
const std::string &group_id,
const VecTensorList &tensor_list) {
TimeRecordWrapper rc("add_vector_batch()");
SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = "
<< tensor_list.tensor_list.size();
BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list);
BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return);
VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance();
scheduler.ExecuteTask(task_ptr);
}
void
VecServiceHandler::add_binary_vector(const std::string& group_id,
VecServiceHandler::add_binary_vector(std::string& _return,
const std::string& group_id,
const VecBinaryTensor& tensor) {
TimeRecordWrapper rc("add_binary_vector()");
SERVER_LOG_TRACE << "group_id = " << group_id << ", vector size = " << tensor.tensor.size()/4;
BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor);
BaseTaskPtr task_ptr = AddVectorTask::Create(group_id, &tensor, _return);
VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance();
scheduler.ExecuteTask(task_ptr);
}
void
VecServiceHandler::add_binary_vector_batch(const std::string& group_id,
VecServiceHandler::add_binary_vector_batch(std::vector<std::string> & _return,
const std::string& group_id,
const VecBinaryTensorList& tensor_list) {
TimeRecordWrapper rc("add_binary_vector_batch()");
SERVER_LOG_TRACE << "group_id = " << group_id << ", vector list size = "
<< tensor_list.tensor_list.size();
BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list);
BaseTaskPtr task_ptr = AddBatchVectorTask::Create(group_id, &tensor_list, _return);
VecServiceScheduler& scheduler = VecServiceScheduler::GetInstance();
scheduler.ExecuteTask(task_ptr);
}
......
cpp/src/server/VecServiceHandler.h
浏览文件 @ faa2c4b4
......@@ -40,19 +40,19 @@ public:
void del_group(const std::string& group_id);
/**
* insert vector interfaces
*
*
* @param group_id
* @param tensor
*/
void add_vector(const std::string& group_id, const VecTensor& tensor);
* insert vector interfaces
*
*
* @param group_id
* @param tensor
*/
void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor);
void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list);
void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list);
void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor);
void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor);
void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list);
void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list);
/**
* search interfaces
......
cpp/src/server/VecServiceTask.cpp
浏览文件 @ faa2c4b4
......@@ -162,31 +162,37 @@ ServerError DeleteGroupTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
AddVectorTask::AddVectorTask(const std::string& group_id,
const VecTensor* tensor)
const VecTensor* tensor,
std::string& id)
: BaseTask(DDL_DML_TASK_GROUP),
group_id_(group_id),
tensor_(tensor),
bin_tensor_(nullptr){
bin_tensor_(nullptr),
tensor_id_(id) {
}
BaseTaskPtr AddVectorTask::Create(const std::string& group_id,
const VecTensor* tensor) {
return std::shared_ptr<BaseTask>(new AddVectorTask(group_id, tensor));
const VecTensor* tensor,
std::string& id) {
return std::shared_ptr<BaseTask>(new AddVectorTask(group_id, tensor, id));
}
AddVectorTask::AddVectorTask(const std::string& group_id,
const VecBinaryTensor* tensor)
const VecBinaryTensor* tensor,
std::string& id)
: BaseTask(DDL_DML_TASK_GROUP),
group_id_(group_id),
tensor_(nullptr),
bin_tensor_(tensor) {
bin_tensor_(tensor),
tensor_id_(id) {
}
BaseTaskPtr AddVectorTask::Create(const std::string& group_id,
const VecBinaryTensor* tensor) {
return std::shared_ptr<BaseTask>(new AddVectorTask(group_id, tensor));
const VecBinaryTensor* tensor,
std::string& id) {
return std::shared_ptr<BaseTask>(new AddVectorTask(group_id, tensor, id));
}
......@@ -265,9 +271,16 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
} else {
std::string uid = GetVecID();
std::string nid = group_id_ + "_" + std::to_string(vector_ids[0]);
std::string num_id = std::to_string(vector_ids[0]);
if(uid.empty()) {
tensor_id_ = num_id;
} else {
tensor_id_ = uid;
}
std::string nid = group_id_ + "_" + num_id;
AttribMap attrib = GetVecAttrib();
attrib[VECTOR_UID] = uid;
attrib[VECTOR_UID] = tensor_id_;
std::string attrib_str;
AttributeSerializer::Encode(attrib, attrib_str);
IVecIdMapper::GetInstance()->Put(nid, attrib_str);
......@@ -286,31 +299,39 @@ ServerError AddVectorTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
AddBatchVectorTask::AddBatchVectorTask(const std::string& group_id,
const VecTensorList* tensor_list)
const VecTensorList* tensor_list,
std::vector<std::string>& ids)
: BaseTask(DDL_DML_TASK_GROUP),
group_id_(group_id),
tensor_list_(tensor_list),
bin_tensor_list_(nullptr) {
bin_tensor_list_(nullptr),
tensor_ids_(ids) {
tensor_ids_.clear();
tensor_ids_.resize(tensor_list->tensor_list.size());
}
BaseTaskPtr AddBatchVectorTask::Create(const std::string& group_id,
const VecTensorList* tensor_list) {
return std::shared_ptr<BaseTask>(new AddBatchVectorTask(group_id, tensor_list));
const VecTensorList* tensor_list,
std::vector<std::string>& ids) {
return std::shared_ptr<BaseTask>(new AddBatchVectorTask(group_id, tensor_list, ids));
}
AddBatchVectorTask::AddBatchVectorTask(const std::string& group_id,
const VecBinaryTensorList* tensor_list)
: BaseTask(DDL_DML_TASK_GROUP),
group_id_(group_id),
tensor_list_(nullptr),
bin_tensor_list_(tensor_list) {
const VecBinaryTensorList* tensor_list,
std::vector<std::string>& ids)
: BaseTask(DDL_DML_TASK_GROUP),
group_id_(group_id),
tensor_list_(nullptr),
bin_tensor_list_(tensor_list),
tensor_ids_(ids) {
tensor_ids_.clear();
tensor_ids_.resize(tensor_list->tensor_list.size());
}
BaseTaskPtr AddBatchVectorTask::Create(const std::string& group_id,
const VecBinaryTensorList* tensor_list) {
return std::shared_ptr<BaseTask>(new AddBatchVectorTask(group_id, tensor_list));
const VecBinaryTensorList* tensor_list,
std::vector<std::string>& ids) {
return std::shared_ptr<BaseTask>(new AddBatchVectorTask(group_id, tensor_list, ids));
}
uint64_t AddBatchVectorTask::GetVecListCount() const {
......@@ -379,11 +400,19 @@ const AttribMap& AddBatchVectorTask::GetVecAttrib(uint64_t index) const {
}
}
void AddBatchVectorTask::ProcessIdMapping(engine::IDNumbers& vector_ids, uint64_t from, uint64_t to) {
void AddBatchVectorTask::ProcessIdMapping(engine::IDNumbers& vector_ids,
uint64_t from, uint64_t to,
std::vector<std::string>& tensor_ids) {
std::string nid_prefix = group_id_ + "_";
for(size_t i = from; i < to; i++) {
std::string uid = GetVecID(i);
std::string nid = nid_prefix + std::to_string(vector_ids[i]);
std::string num_id = std::to_string(vector_ids[i]);
if(uid.empty()) {
uid = num_id;
}
tensor_ids_[i] = uid;
std::string nid = nid_prefix + num_id;
AttribMap attrib = GetVecAttrib(i);
attrib[VECTOR_UID] = uid;
std::string attrib_str;
......@@ -437,7 +466,7 @@ ServerError AddBatchVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
} else {
if(vec_count < USE_MT) {
ProcessIdMapping(vector_ids, 0, vec_count);
ProcessIdMapping(vector_ids, 0, vec_count, tensor_ids_);
rc.Record("built id mapping");
} else {
std::list<std::future<void>> threads_list;
......@@ -446,7 +475,7 @@ ServerError AddBatchVectorTask::OnExecute() {
while(end_index < vec_count) {
threads_list.push_back(
GetThreadPool().enqueue(&AddBatchVectorTask::ProcessIdMapping,
this, vector_ids, begin_index, end_index));
this, vector_ids, begin_index, end_index, tensor_ids_));
begin_index = end_index;
end_index += USE_MT;
if(end_index > vec_count) {
......
cpp/src/server/VecServiceTask.h
浏览文件 @ faa2c4b4
......@@ -71,17 +71,21 @@ private:
class AddVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& group_id,
const VecTensor* tensor);
const VecTensor* tensor,
std::string& id);
static BaseTaskPtr Create(const std::string& group_id,
const VecBinaryTensor* tensor);
const VecBinaryTensor* tensor,
std::string& id);
protected:
AddVectorTask(const std::string& group_id,
const VecTensor* tensor);
const VecTensor* tensor,
std::string& id);
AddVectorTask(const std::string& group_id,
const VecBinaryTensor* tensor);
const VecBinaryTensor* tensor,
std::string& id);
uint64_t GetVecDimension() const;
const double* GetVecData() const;
......@@ -94,6 +98,7 @@ private:
std::string group_id_;
const VecTensor* tensor_;
const VecBinaryTensor* bin_tensor_;
std::string& tensor_id_;
};
......@@ -101,17 +106,21 @@ private:
class AddBatchVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& group_id,
const VecTensorList* tensor_list);
const VecTensorList* tensor_list,
std::vector<std::string>& ids);
static BaseTaskPtr Create(const std::string& group_id,
const VecBinaryTensorList* tensor_list);
const VecBinaryTensorList* tensor_list,
std::vector<std::string>& ids);
protected:
AddBatchVectorTask(const std::string& group_id,
const VecTensorList* tensor_list);
const VecTensorList* tensor_list,
std::vector<std::string>& ids);
AddBatchVectorTask(const std::string& group_id,
const VecBinaryTensorList* tensor_list);
const VecBinaryTensorList* tensor_list,
std::vector<std::string>& ids);
uint64_t GetVecListCount() const;
uint64_t GetVecDimension(uint64_t index) const;
......@@ -119,7 +128,9 @@ protected:
std::string GetVecID(uint64_t index) const;
const AttribMap& GetVecAttrib(uint64_t index) const;
void ProcessIdMapping(engine::IDNumbers& vector_ids, uint64_t from, uint64_t to);
void ProcessIdMapping(engine::IDNumbers& vector_ids,
uint64_t from, uint64_t to,
std::vector<std::string>& tensor_ids);
ServerError OnExecute() override;
......@@ -127,6 +138,7 @@ private:
std::string group_id_;
const VecTensorList* tensor_list_;
const VecBinaryTensorList* bin_tensor_list_;
std::vector<std::string>& tensor_ids_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......
cpp/src/thrift/VectorService.thrift
浏览文件 @ faa2c4b4
......@@ -110,10 +110,10 @@ service VecService {
* insert vector interfaces
*
*/
void add_vector(2: string group_id, 3: VecTensor tensor) throws(1: VecException e);
void add_vector_batch(2: string group_id, 3: VecTensorList tensor_list) throws(1: VecException e);
void add_binary_vector(2: string group_id, 3: VecBinaryTensor tensor) throws(1: VecException e);
void add_binary_vector_batch(2: string group_id, 3: VecBinaryTensorList tensor_list) throws(1: VecException e);
string add_vector(2: string group_id, 3: VecTensor tensor) throws(1: VecException e);
list<string> add_vector_batch(2: string group_id, 3: VecTensorList tensor_list) throws(1: VecException e);
string add_binary_vector(2: string group_id, 3: VecBinaryTensor tensor) throws(1: VecException e);
list<string> add_binary_vector_batch(2: string group_id, 3: VecBinaryTensorList tensor_list) throws(1: VecException e);
/**
* search interfaces
......
cpp/src/thrift/gen-cpp/VecService.h
浏览文件 @ faa2c4b4
......@@ -38,10 +38,10 @@ class VecServiceIf {
* @param group_id
* @param tensor
*/
virtual void add_vector(const std::string& group_id, const VecTensor& tensor) = 0;
virtual void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list) = 0;
virtual void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor) = 0;
virtual void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list) = 0;
virtual void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor) = 0;
virtual void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list) = 0;
virtual void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor) = 0;
virtual void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list) = 0;
/**
* search interfaces
......@@ -98,16 +98,16 @@ class VecServiceNull : virtual public VecServiceIf {
void del_group(const std::string& /* group_id */) {
return;
}
void add_vector(const std::string& /* group_id */, const VecTensor& /* tensor */) {
void add_vector(std::string& /* _return */, const std::string& /* group_id */, const VecTensor& /* tensor */) {
return;
}
void add_vector_batch(const std::string& /* group_id */, const VecTensorList& /* tensor_list */) {
void add_vector_batch(std::vector<std::string> & /* _return */, const std::string& /* group_id */, const VecTensorList& /* tensor_list */) {
return;
}
void add_binary_vector(const std::string& /* group_id */, const VecBinaryTensor& /* tensor */) {
void add_binary_vector(std::string& /* _return */, const std::string& /* group_id */, const VecBinaryTensor& /* tensor */) {
return;
}
void add_binary_vector_batch(const std::string& /* group_id */, const VecBinaryTensorList& /* tensor_list */) {
void add_binary_vector_batch(std::vector<std::string> & /* _return */, const std::string& /* group_id */, const VecBinaryTensorList& /* tensor_list */) {
return;
}
void search_vector(VecSearchResult& /* _return */, const std::string& /* group_id */, const int64_t /* top_k */, const VecTensor& /* tensor */, const VecSearchFilter& /* filter */) {
......@@ -501,7 +501,8 @@ class VecService_add_vector_pargs {
};
typedef struct _VecService_add_vector_result__isset {
_VecService_add_vector_result__isset() : e(false) {}
_VecService_add_vector_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_vector_result__isset;
......@@ -510,18 +511,23 @@ class VecService_add_vector_result {
VecService_add_vector_result(const VecService_add_vector_result&);
VecService_add_vector_result& operator=(const VecService_add_vector_result&);
VecService_add_vector_result() {
VecService_add_vector_result() : success() {
}
virtual ~VecService_add_vector_result() throw();
std::string success;
VecException e;
_VecService_add_vector_result__isset __isset;
void __set_success(const std::string& val);
void __set_e(const VecException& val);
bool operator == (const VecService_add_vector_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
......@@ -538,7 +544,8 @@ class VecService_add_vector_result {
};
typedef struct _VecService_add_vector_presult__isset {
_VecService_add_vector_presult__isset() : e(false) {}
_VecService_add_vector_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_vector_presult__isset;
......@@ -547,6 +554,7 @@ class VecService_add_vector_presult {
virtual ~VecService_add_vector_presult() throw();
std::string* success;
VecException e;
_VecService_add_vector_presult__isset __isset;
......@@ -612,7 +620,8 @@ class VecService_add_vector_batch_pargs {
};
typedef struct _VecService_add_vector_batch_result__isset {
_VecService_add_vector_batch_result__isset() : e(false) {}
_VecService_add_vector_batch_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_vector_batch_result__isset;
......@@ -625,14 +634,19 @@ class VecService_add_vector_batch_result {
}
virtual ~VecService_add_vector_batch_result() throw();
std::vector<std::string> success;
VecException e;
_VecService_add_vector_batch_result__isset __isset;
void __set_success(const std::vector<std::string> & val);
void __set_e(const VecException& val);
bool operator == (const VecService_add_vector_batch_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
......@@ -649,7 +663,8 @@ class VecService_add_vector_batch_result {
};
typedef struct _VecService_add_vector_batch_presult__isset {
_VecService_add_vector_batch_presult__isset() : e(false) {}
_VecService_add_vector_batch_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_vector_batch_presult__isset;
......@@ -658,6 +673,7 @@ class VecService_add_vector_batch_presult {
virtual ~VecService_add_vector_batch_presult() throw();
std::vector<std::string> * success;
VecException e;
_VecService_add_vector_batch_presult__isset __isset;
......@@ -723,7 +739,8 @@ class VecService_add_binary_vector_pargs {
};
typedef struct _VecService_add_binary_vector_result__isset {
_VecService_add_binary_vector_result__isset() : e(false) {}
_VecService_add_binary_vector_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_binary_vector_result__isset;
......@@ -732,18 +749,23 @@ class VecService_add_binary_vector_result {
VecService_add_binary_vector_result(const VecService_add_binary_vector_result&);
VecService_add_binary_vector_result& operator=(const VecService_add_binary_vector_result&);
VecService_add_binary_vector_result() {
VecService_add_binary_vector_result() : success() {
}
virtual ~VecService_add_binary_vector_result() throw();
std::string success;
VecException e;
_VecService_add_binary_vector_result__isset __isset;
void __set_success(const std::string& val);
void __set_e(const VecException& val);
bool operator == (const VecService_add_binary_vector_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
......@@ -760,7 +782,8 @@ class VecService_add_binary_vector_result {
};
typedef struct _VecService_add_binary_vector_presult__isset {
_VecService_add_binary_vector_presult__isset() : e(false) {}
_VecService_add_binary_vector_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_binary_vector_presult__isset;
......@@ -769,6 +792,7 @@ class VecService_add_binary_vector_presult {
virtual ~VecService_add_binary_vector_presult() throw();
std::string* success;
VecException e;
_VecService_add_binary_vector_presult__isset __isset;
......@@ -834,7 +858,8 @@ class VecService_add_binary_vector_batch_pargs {
};
typedef struct _VecService_add_binary_vector_batch_result__isset {
_VecService_add_binary_vector_batch_result__isset() : e(false) {}
_VecService_add_binary_vector_batch_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_binary_vector_batch_result__isset;
......@@ -847,14 +872,19 @@ class VecService_add_binary_vector_batch_result {
}
virtual ~VecService_add_binary_vector_batch_result() throw();
std::vector<std::string> success;
VecException e;
_VecService_add_binary_vector_batch_result__isset __isset;
void __set_success(const std::vector<std::string> & val);
void __set_e(const VecException& val);
bool operator == (const VecService_add_binary_vector_batch_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
......@@ -871,7 +901,8 @@ class VecService_add_binary_vector_batch_result {
};
typedef struct _VecService_add_binary_vector_batch_presult__isset {
_VecService_add_binary_vector_batch_presult__isset() : e(false) {}
_VecService_add_binary_vector_batch_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _VecService_add_binary_vector_batch_presult__isset;
......@@ -880,6 +911,7 @@ class VecService_add_binary_vector_batch_presult {
virtual ~VecService_add_binary_vector_batch_presult() throw();
std::vector<std::string> * success;
VecException e;
_VecService_add_binary_vector_batch_presult__isset __isset;
......@@ -1454,18 +1486,18 @@ class VecServiceClient : virtual public VecServiceIf {
void del_group(const std::string& group_id);
void send_del_group(const std::string& group_id);
void recv_del_group();
void add_vector(const std::string& group_id, const VecTensor& tensor);
void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor);
void send_add_vector(const std::string& group_id, const VecTensor& tensor);
void recv_add_vector();
void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list);
void recv_add_vector(std::string& _return);
void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list);
void send_add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list);
void recv_add_vector_batch();
void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor);
void recv_add_vector_batch(std::vector<std::string> & _return);
void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor);
void send_add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor);
void recv_add_binary_vector();
void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list);
void recv_add_binary_vector(std::string& _return);
void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list);
void send_add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list);
void recv_add_binary_vector_batch();
void recv_add_binary_vector_batch(std::vector<std::string> & _return);
void search_vector(VecSearchResult& _return, const std::string& group_id, const int64_t top_k, const VecTensor& tensor, const VecSearchFilter& filter);
void send_search_vector(const std::string& group_id, const int64_t top_k, const VecTensor& tensor, const VecSearchFilter& filter);
void recv_search_vector(VecSearchResult& _return);
......@@ -1574,40 +1606,44 @@ class VecServiceMultiface : virtual public VecServiceIf {
ifaces_[i]->del_group(group_id);
}
void add_vector(const std::string& group_id, const VecTensor& tensor) {
void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->add_vector(group_id, tensor);
ifaces_[i]->add_vector(_return, group_id, tensor);
}
ifaces_[i]->add_vector(group_id, tensor);
ifaces_[i]->add_vector(_return, group_id, tensor);
return;
}
void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list) {
void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->add_vector_batch(group_id, tensor_list);
ifaces_[i]->add_vector_batch(_return, group_id, tensor_list);
}
ifaces_[i]->add_vector_batch(group_id, tensor_list);
ifaces_[i]->add_vector_batch(_return, group_id, tensor_list);
return;
}
void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor) {
void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->add_binary_vector(group_id, tensor);
ifaces_[i]->add_binary_vector(_return, group_id, tensor);
}
ifaces_[i]->add_binary_vector(group_id, tensor);
ifaces_[i]->add_binary_vector(_return, group_id, tensor);
return;
}
void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list) {
void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->add_binary_vector_batch(group_id, tensor_list);
ifaces_[i]->add_binary_vector_batch(_return, group_id, tensor_list);
}
ifaces_[i]->add_binary_vector_batch(group_id, tensor_list);
ifaces_[i]->add_binary_vector_batch(_return, group_id, tensor_list);
return;
}
void search_vector(VecSearchResult& _return, const std::string& group_id, const int64_t top_k, const VecTensor& tensor, const VecSearchFilter& filter) {
......@@ -1689,18 +1725,18 @@ class VecServiceConcurrentClient : virtual public VecServiceIf {
void del_group(const std::string& group_id);
int32_t send_del_group(const std::string& group_id);
void recv_del_group(const int32_t seqid);
void add_vector(const std::string& group_id, const VecTensor& tensor);
void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor);
int32_t send_add_vector(const std::string& group_id, const VecTensor& tensor);
void recv_add_vector(const int32_t seqid);
void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list);
void recv_add_vector(std::string& _return, const int32_t seqid);
void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list);
int32_t send_add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list);
void recv_add_vector_batch(const int32_t seqid);
void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor);
void recv_add_vector_batch(std::vector<std::string> & _return, const int32_t seqid);
void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor);
int32_t send_add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor);
void recv_add_binary_vector(const int32_t seqid);
void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list);
void recv_add_binary_vector(std::string& _return, const int32_t seqid);
void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list);
int32_t send_add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list);
void recv_add_binary_vector_batch(const int32_t seqid);
void recv_add_binary_vector_batch(std::vector<std::string> & _return, const int32_t seqid);
void search_vector(VecSearchResult& _return, const std::string& group_id, const int64_t top_k, const VecTensor& tensor, const VecSearchFilter& filter);
int32_t send_search_vector(const std::string& group_id, const int64_t top_k, const VecTensor& tensor, const VecSearchFilter& filter);
void recv_search_vector(VecSearchResult& _return, const int32_t seqid);
......
cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp
浏览文件 @ faa2c4b4
......@@ -47,22 +47,22 @@ class VecServiceHandler : virtual public VecServiceIf {
* @param group_id
* @param tensor
*/
void add_vector(const std::string& group_id, const VecTensor& tensor) {
void add_vector(std::string& _return, const std::string& group_id, const VecTensor& tensor) {
// Your implementation goes here
printf("add_vector\n");
}
void add_vector_batch(const std::string& group_id, const VecTensorList& tensor_list) {
void add_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecTensorList& tensor_list) {
// Your implementation goes here
printf("add_vector_batch\n");
}
void add_binary_vector(const std::string& group_id, const VecBinaryTensor& tensor) {
void add_binary_vector(std::string& _return, const std::string& group_id, const VecBinaryTensor& tensor) {
// Your implementation goes here
printf("add_binary_vector\n");
}
void add_binary_vector_batch(const std::string& group_id, const VecBinaryTensorList& tensor_list) {
void add_binary_vector_batch(std::vector<std::string> & _return, const std::string& group_id, const VecBinaryTensorList& tensor_list) {
// Your implementation goes here
printf("add_binary_vector_batch\n");
}
......
cpp/src/thrift/gen-py/zilliz/VecService-remote
浏览文件 @ faa2c4b4
......@@ -27,10 +27,10 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
print(' void add_group(VecGroup group)')
print(' VecGroup get_group(string group_id)')
print(' void del_group(string group_id)')
print(' void add_vector(string group_id, VecTensor tensor)')
print(' void add_vector_batch(string group_id, VecTensorList tensor_list)')
print(' void add_binary_vector(string group_id, VecBinaryTensor tensor)')
print(' void add_binary_vector_batch(string group_id, VecBinaryTensorList tensor_list)')
print(' string add_vector(string group_id, VecTensor tensor)')
print(' add_vector_batch(string group_id, VecTensorList tensor_list)')
print(' string add_binary_vector(string group_id, VecBinaryTensor tensor)')
print(' add_binary_vector_batch(string group_id, VecBinaryTensorList tensor_list)')
print(' VecSearchResult search_vector(string group_id, i64 top_k, VecTensor tensor, VecSearchFilter filter)')
print(' VecSearchResultList search_vector_batch(string group_id, i64 top_k, VecTensorList tensor_list, VecSearchFilter filter)')
print(' VecSearchResult search_binary_vector(string group_id, i64 top_k, VecBinaryTensor tensor, VecSearchFilter filter)')
......
cpp/src/thrift/gen-py/zilliz/VecService.py
浏览文件 @ faa2c4b4
......@@ -254,7 +254,7 @@ class Client(Iface):
"""
self.send_add_vector(group_id, tensor)
self.recv_add_vector()
return self.recv_add_vector()
def send_add_vector(self, group_id, tensor):
self._oprot.writeMessageBegin('add_vector', TMessageType.CALL, self._seqid)
......@@ -276,9 +276,11 @@ class Client(Iface):
result = add_vector_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
if result.e is not None:
raise result.e
return
raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector failed: unknown result")
def add_vector_batch(self, group_id, tensor_list):
"""
......@@ -288,7 +290,7 @@ class Client(Iface):
"""
self.send_add_vector_batch(group_id, tensor_list)
self.recv_add_vector_batch()
return self.recv_add_vector_batch()
def send_add_vector_batch(self, group_id, tensor_list):
self._oprot.writeMessageBegin('add_vector_batch', TMessageType.CALL, self._seqid)
......@@ -310,9 +312,11 @@ class Client(Iface):
result = add_vector_batch_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
if result.e is not None:
raise result.e
return
raise TApplicationException(TApplicationException.MISSING_RESULT, "add_vector_batch failed: unknown result")
def add_binary_vector(self, group_id, tensor):
"""
......@@ -322,7 +326,7 @@ class Client(Iface):
"""
self.send_add_binary_vector(group_id, tensor)
self.recv_add_binary_vector()
return self.recv_add_binary_vector()
def send_add_binary_vector(self, group_id, tensor):
self._oprot.writeMessageBegin('add_binary_vector', TMessageType.CALL, self._seqid)
......@@ -344,9 +348,11 @@ class Client(Iface):
result = add_binary_vector_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
if result.e is not None:
raise result.e
return
raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector failed: unknown result")
def add_binary_vector_batch(self, group_id, tensor_list):
"""
......@@ -356,7 +362,7 @@ class Client(Iface):
"""
self.send_add_binary_vector_batch(group_id, tensor_list)
self.recv_add_binary_vector_batch()
return self.recv_add_binary_vector_batch()
def send_add_binary_vector_batch(self, group_id, tensor_list):
self._oprot.writeMessageBegin('add_binary_vector_batch', TMessageType.CALL, self._seqid)
......@@ -378,9 +384,11 @@ class Client(Iface):
result = add_binary_vector_batch_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
if result.e is not None:
raise result.e
return
raise TApplicationException(TApplicationException.MISSING_RESULT, "add_binary_vector_batch failed: unknown result")
def search_vector(self, group_id, top_k, tensor, filter):
"""
......@@ -665,7 +673,7 @@ class Processor(Iface, TProcessor):
iprot.readMessageEnd()
result = add_vector_result()
try:
self._handler.add_vector(args.group_id, args.tensor)
result.success = self._handler.add_vector(args.group_id, args.tensor)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
......@@ -691,7 +699,7 @@ class Processor(Iface, TProcessor):
iprot.readMessageEnd()
result = add_vector_batch_result()
try:
self._handler.add_vector_batch(args.group_id, args.tensor_list)
result.success = self._handler.add_vector_batch(args.group_id, args.tensor_list)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
......@@ -717,7 +725,7 @@ class Processor(Iface, TProcessor):
iprot.readMessageEnd()
result = add_binary_vector_result()
try:
self._handler.add_binary_vector(args.group_id, args.tensor)
result.success = self._handler.add_binary_vector(args.group_id, args.tensor)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
......@@ -743,7 +751,7 @@ class Processor(Iface, TProcessor):
iprot.readMessageEnd()
result = add_binary_vector_batch_result()
try:
self._handler.add_binary_vector_batch(args.group_id, args.tensor_list)
result.success = self._handler.add_binary_vector_batch(args.group_id, args.tensor_list)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
......@@ -1340,12 +1348,14 @@ add_vector_args.thrift_spec = (
class add_vector_result(object):
"""
Attributes:
- success
- e
"""
def __init__(self, e=None,):
def __init__(self, success=None, e=None,):
self.success = success
self.e = e
def read(self, iprot):
......@@ -1357,7 +1367,12 @@ class add_vector_result(object):
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if fid == 0:
if ftype == TType.STRING:
self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
elif fid == 1:
if ftype == TType.STRUCT:
self.e = VecException()
self.e.read(iprot)
......@@ -1373,6 +1388,10 @@ class add_vector_result(object):
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('add_vector_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.STRING, 0)
oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success)
oprot.writeFieldEnd()
if self.e is not None:
oprot.writeFieldBegin('e', TType.STRUCT, 1)
self.e.write(oprot)
......@@ -1395,7 +1414,7 @@ class add_vector_result(object):
return not (self == other)
all_structs.append(add_vector_result)
add_vector_result.thrift_spec = (
None, # 0
(0, TType.STRING, 'success', 'UTF8', None, ), # 0
(1, TType.STRUCT, 'e', [VecException, None], None, ), # 1
)
......@@ -1479,12 +1498,14 @@ add_vector_batch_args.thrift_spec = (
class add_vector_batch_result(object):
"""
Attributes:
- success
- e
"""
def __init__(self, e=None,):
def __init__(self, success=None, e=None,):
self.success = success
self.e = e
def read(self, iprot):
......@@ -1496,7 +1517,17 @@ class add_vector_batch_result(object):
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if fid == 0:
if ftype == TType.LIST:
self.success = []
(_etype81, _size78) = iprot.readListBegin()
for _i82 in range(_size78):
_elem83 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
self.success.append(_elem83)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 1:
if ftype == TType.STRUCT:
self.e = VecException()
self.e.read(iprot)
......@@ -1512,6 +1543,13 @@ class add_vector_batch_result(object):
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('add_vector_batch_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.LIST, 0)
oprot.writeListBegin(TType.STRING, len(self.success))
for iter84 in self.success:
oprot.writeString(iter84.encode('utf-8') if sys.version_info[0] == 2 else iter84)
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.e is not None:
oprot.writeFieldBegin('e', TType.STRUCT, 1)
self.e.write(oprot)
......@@ -1534,7 +1572,7 @@ class add_vector_batch_result(object):
return not (self == other)
all_structs.append(add_vector_batch_result)
add_vector_batch_result.thrift_spec = (
None, # 0
(0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0
(1, TType.STRUCT, 'e', [VecException, None], None, ), # 1
)
......@@ -1618,12 +1656,14 @@ add_binary_vector_args.thrift_spec = (
class add_binary_vector_result(object):
"""
Attributes:
- success
- e
"""
def __init__(self, e=None,):
def __init__(self, success=None, e=None,):
self.success = success
self.e = e
def read(self, iprot):
......@@ -1635,7 +1675,12 @@ class add_binary_vector_result(object):
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if fid == 0:
if ftype == TType.STRING:
self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
elif fid == 1:
if ftype == TType.STRUCT:
self.e = VecException()
self.e.read(iprot)
......@@ -1651,6 +1696,10 @@ class add_binary_vector_result(object):
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('add_binary_vector_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.STRING, 0)
oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success)
oprot.writeFieldEnd()
if self.e is not None:
oprot.writeFieldBegin('e', TType.STRUCT, 1)
self.e.write(oprot)
......@@ -1673,7 +1722,7 @@ class add_binary_vector_result(object):
return not (self == other)
all_structs.append(add_binary_vector_result)
add_binary_vector_result.thrift_spec = (
None, # 0
(0, TType.STRING, 'success', 'UTF8', None, ), # 0
(1, TType.STRUCT, 'e', [VecException, None], None, ), # 1
)
......@@ -1757,12 +1806,14 @@ add_binary_vector_batch_args.thrift_spec = (
class add_binary_vector_batch_result(object):
"""
Attributes:
- success
- e
"""
def __init__(self, e=None,):
def __init__(self, success=None, e=None,):
self.success = success
self.e = e
def read(self, iprot):
......@@ -1774,7 +1825,17 @@ class add_binary_vector_batch_result(object):
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if fid == 0:
if ftype == TType.LIST:
self.success = []
(_etype88, _size85) = iprot.readListBegin()
for _i89 in range(_size85):
_elem90 = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
self.success.append(_elem90)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 1:
if ftype == TType.STRUCT:
self.e = VecException()
self.e.read(iprot)
......@@ -1790,6 +1851,13 @@ class add_binary_vector_batch_result(object):
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('add_binary_vector_batch_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.LIST, 0)
oprot.writeListBegin(TType.STRING, len(self.success))
for iter91 in self.success:
oprot.writeString(iter91.encode('utf-8') if sys.version_info[0] == 2 else iter91)
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.e is not None:
oprot.writeFieldBegin('e', TType.STRUCT, 1)
self.e.write(oprot)
......@@ -1812,7 +1880,7 @@ class add_binary_vector_batch_result(object):
return not (self == other)
all_structs.append(add_binary_vector_batch_result)
add_binary_vector_batch_result.thrift_spec = (
None, # 0
(0, TType.LIST, 'success', (TType.STRING, 'UTF8', False), None, ), # 0
(1, TType.STRUCT, 'e', [VecException, None], None, ), # 1
)
......
cpp/test_client/src/ClientTest.cpp
浏览文件 @ faa2c4b4
......@@ -21,6 +21,7 @@ using namespace zilliz::vecwise::client;
namespace {
static const int32_t VEC_DIMENSION = 256;
static const int64_t BATCH_COUNT = 1000;
static const std::string TEST_ATTRIB_NUM = "number";
static const std::string TEST_ATTRIB_COMMENT = "comment";
......@@ -94,14 +95,14 @@ namespace {
if(tensor_list) {
tensor.uid = "normal_vec_" + std::to_string(k);
attrib_map[TEST_ATTRIB_COMMENT] = tensor.uid;
attrib_map[TEST_ATTRIB_COMMENT] = "this is vector " + tensor.uid;
tensor.__set_attrib(attrib_map);
tensor_list->tensor_list.emplace_back(tensor);
}
if(bin_tensor_list) {
bin_tensor.uid = "binary_vec_" + std::to_string(k);
attrib_map[TEST_ATTRIB_COMMENT] = bin_tensor.uid;
attrib_map[TEST_ATTRIB_COMMENT] = "this is binary vector " + bin_tensor.uid;
bin_tensor.__set_attrib(attrib_map);
bin_tensor_list->tensor_list.emplace_back(bin_tensor);
}
......@@ -116,55 +117,56 @@ namespace {
}
}
void ClientTest::LoopTest() {
server::TimeRecorder rc("LoopTest");
std::string address, protocol;
int32_t port = 0;
GetServerAddress(address, port, protocol);
client::ClientSession session(address, port, protocol);
rc.Record("connection");
//add group
VecGroup group;
group.id = "loop_group";
group.dimension = VEC_DIMENSION;
group.index_type = 0;
session.interface()->add_group(group);
rc.Record("add group");
const int64_t batch = 10000;
for(int64_t i = 0; i < 1000; i++) {
{
VecBinaryTensorList bin_tensor_list;
BuildVectors(i * batch, (i + 1) * batch, nullptr, &bin_tensor_list);
rc.Record("build batch no." + std::to_string(i));
session.interface()->add_binary_vector_batch(group.id, bin_tensor_list);
rc.Record("add batch no." + std::to_string(i));
}
sleep(1);
rc.Record("sleep 1 second");
VecTensor tensor;
for (int32_t k = 0; k < VEC_DIMENSION; k++) {
tensor.tensor.push_back((double) (k + i*666));
}
//do search
VecSearchResult res;
VecSearchFilter filter;
session.interface()->search_vector(res, group.id, 10, tensor, filter);
rc.Record("search finish");
std::cout << "Search result: " << std::endl;
for(VecSearchResultItem& item : res.result_list) {
std::cout << "\t" << item.uid << std::endl;
}
}
}
//void ClientTest::LoopTest() {
// server::TimeRecorder rc("LoopTest");
//
// std::string address, protocol;
// int32_t port = 0;
// GetServerAddress(address, port, protocol);
// client::ClientSession session(address, port, protocol);
//
// rc.Record("connection");
//
// //add group
// VecGroup group;
// group.id = "loop_group";
// group.dimension = VEC_DIMENSION;
// group.index_type = 0;
// session.interface()->add_group(group);
// rc.Record("add group");
//
// const int64_t batch = 10000;
// for(int64_t i = 0; i < 1000; i++) {
// {
// VecBinaryTensorList bin_tensor_list;
// BuildVectors(i * batch, (i + 1) * batch, nullptr, &bin_tensor_list);
// rc.Record("build batch no." + std::to_string(i));
//
// std::vector<std::string> ids;
// session.interface()->add_binary_vector_batch(ids, group.id, bin_tensor_list);
// rc.Record("add batch no." + std::to_string(i));
// }
//
// sleep(1);
// rc.Record("sleep 1 second");
//
// VecTensor tensor;
// for (int32_t k = 0; k < VEC_DIMENSION; k++) {
// tensor.tensor.push_back((double) (k + i*666));
// }
//
// //do search
// VecSearchResult res;
// VecSearchFilter filter;
// session.interface()->search_vector(res, group.id, 10, tensor, filter);
// rc.Record("search finish");
//
// std::cout << "Search result: " << std::endl;
// for(VecSearchResultItem& item : res.result_list) {
// std::cout << "\t" << item.uid << std::endl;
// }
// }
//}
TEST(AddVector, CLIENT_TEST) {
try {
......@@ -182,47 +184,57 @@ TEST(AddVector, CLIENT_TEST) {
//prepare data
CLIENT_LOG_INFO << "Preparing vectors...";
const int64_t count = 100000;
VecTensorList tensor_list;
VecBinaryTensorList bin_tensor_list;
BuildVectors(0, count, &tensor_list, &bin_tensor_list);
// //add vectors one by one
// {
// server::TimeRecorder rc("Add " + std::to_string(count) + " vectors one by one");
// for (int64_t k = 0; k < count; k++) {
// session.interface()->add_vector(group.id, tensor_list.tensor_list[k]);
// if (k % 1000 == 0) {
// CLIENT_LOG_INFO << "add normal vector no." << k;
// }
// }
// rc.Elapse("done!");
// }
//
// //add vectors in one batch
// {
// server::TimeRecorder rc("Add " + std::to_string(count) + " vectors in one batch");
// session.interface()->add_vector_batch(group.id, tensor_list);
// rc.Elapse("done!");
// }
const int64_t count = BATCH_COUNT;
VecTensorList tensor_list_1, tensor_list_2;
VecBinaryTensorList bin_tensor_list_1, bin_tensor_list_2;
BuildVectors(0, count, &tensor_list_1, &bin_tensor_list_1);
BuildVectors(count, count*2, &tensor_list_2, &bin_tensor_list_2);
#if 0
//add vectors one by one
{
server::TimeRecorder rc("Add " + std::to_string(count) + " vectors one by one");
for (int64_t k = 0; k < count; k++) {
std::string id;
tensor_list_1.tensor_list[k].uid = "";
session.interface()->add_vector(id, group.id, tensor_list_1.tensor_list[k]);
if (k % 1000 == 0) {
CLIENT_LOG_INFO << "add normal vector no." << k;
}
ASSERT_TRUE(!id.empty());
}
rc.Elapse("done!");
}
//add vectors in one batch
{
server::TimeRecorder rc("Add " + std::to_string(count) + " vectors in one batch");
std::vector<std::string> ids;
session.interface()->add_vector_batch(ids, group.id, tensor_list_2);
rc.Elapse("done!");
}
#else
//add binary vectors one by one
{
server::TimeRecorder rc("Add " + std::to_string(count) + " binary vectors one by one");
for (int64_t k = 0; k < count; k++) {
session.interface()->add_binary_vector(group.id, bin_tensor_list.tensor_list[k]);
std::string id;
bin_tensor_list_1.tensor_list[k].uid = "";
session.interface()->add_binary_vector(id, group.id, bin_tensor_list_1.tensor_list[k]);
if (k % 1000 == 0) {
CLIENT_LOG_INFO << "add binary vector no." << k;
}
ASSERT_TRUE(!id.empty());
}
rc.Elapse("done!");
}
#else
//add binary vectors in one batch
{
server::TimeRecorder rc("Add " + std::to_string(count) + " binary vectors in one batch");
session.interface()->add_binary_vector_batch(group.id, bin_tensor_list);
std::vector<std::string> ids;
session.interface()->add_binary_vector_batch(ids, group.id, bin_tensor_list_2);
rc.Elapse("done!");
}
#endif
......@@ -274,13 +286,13 @@ TEST(SearchVector, CLIENT_TEST) {
ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0);
ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0);
ASSERT_TRUE(item.attrib[TEST_ATTRIB_COMMENT].find(item.uid) != std::string::npos);
ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty());
}
rc.Elapse("done!");
ASSERT_EQ(res.result_list.size(), (uint64_t)top_k);
if(!res.result_list.empty()) {
ASSERT_TRUE(res.result_list[0].uid.find(std::to_string(anchor_index)) != std::string::npos);
ASSERT_TRUE(!res.result_list[0].uid.empty());
}
//empty search
......@@ -297,7 +309,7 @@ TEST(SearchVector, CLIENT_TEST) {
//search binary vector
{
const int32_t anchor_index = 300;
const int32_t anchor_index = BATCH_COUNT + 200;
const int32_t search_count = 10;
const int64_t top_k = 5;
server::TimeRecorder rc("Search binary batch top_k");
......@@ -324,7 +336,7 @@ TEST(SearchVector, CLIENT_TEST) {
std::cout << "\t" << item.uid << std::endl;
ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_NUM) != 0);
ASSERT_TRUE(item.attrib.count(TEST_ATTRIB_COMMENT) != 0);
ASSERT_TRUE(item.attrib[TEST_ATTRIB_COMMENT].find(item.uid) != std::string::npos);
ASSERT_TRUE(!item.attrib[TEST_ATTRIB_COMMENT].empty());
}
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或