提交 8f22ebae 编写于 作者: J Jiawei Wang

fix dist kv

上级 97f0b646
......@@ -74,7 +74,7 @@ int GeneralDistKVInferOp::inference() {
const TensorVector *in = &input_blob->tensor_vector;
TensorVector *out = &output_blob->tensor_vector;
std::vector<uint64_t> keys;
std::vector<uint64_t> rm_dup_keys;
std::vector<uint64_t> unique_keys;
std::unordered_map<uint64_t, rec::mcube::CubeValue*> key_map;
std::vector<rec::mcube::CubeValue> values;
int sparse_count = 0;
......@@ -96,7 +96,7 @@ int GeneralDistKVInferOp::inference() {
dataptr_size_pairs.push_back(std::make_pair(data_ptr, elem_num));
}
keys.resize(key_len);
rm_dup_keys.resize(key_len);
unique_keys.resize(key_len);
int key_idx = 0;
for (size_t i = 0; i < dataptr_size_pairs.size(); ++i) {
std::copy(dataptr_size_pairs[i].first,
......@@ -105,15 +105,15 @@ int GeneralDistKVInferOp::inference() {
key_idx += dataptr_size_pairs[i].second;
}
int rm_dup_keys_count = 0;
int unique_keys_count = 0;
for (size_t i = 0; i < keys.size(); ++i) {
if (key_map.find(keys[i]) == key_map.end()) {
key_map[keys[i]] = nullptr;
rm_dup_keys[rm_dup_keys_count++] = keys[i];
unique_keys[unique_keys_count++] = keys[i];
}
}
rm_dup_keys.resize(rm_dup_keys_count);
VLOG(1) << "(logid=" << log_id << ") cube number of keys to look up: " << key_len << " uniq keys: "<< rm_dup_keys_count;
unique_keys.resize(unique_keys_count);
VLOG(1) << "(logid=" << log_id << ") cube number of keys to look up: " << key_len << " uniq keys: "<< unique_keys_count;
rec::mcube::CubeAPI *cube = rec::mcube::CubeAPI::instance();
std::vector<std::string> table_names = cube->get_table_names();
if (table_names.size() == 0) {
......@@ -121,11 +121,11 @@ int GeneralDistKVInferOp::inference() {
return -1;
}
int64_t seek_start = timeline.TimeStampUS();
int ret = cube->seek(table_names[0], rm_dup_keys, &values);
int ret = cube->seek(table_names[0], unique_keys, &values);
int64_t seek_end = timeline.TimeStampUS();
VLOG(2) << "(logid=" << log_id << ") cube seek status: " << ret << " seek_time: " << seek_end - seek_start;
for (size_t i = 0; i < rm_dup_keys.size(); ++i) {
key_map[rm_dup_keys[i]] = &values[i];
for (size_t i = 0; i < unique_keys.size(); ++i) {
key_map[unique_keys[i]] = &values[i];
}
if (values.size() != keys.size() || values[0].buff.size() == 0) {
LOG(ERROR) << "cube value return null";
......@@ -177,7 +177,7 @@ int GeneralDistKVInferOp::inference() {
continue;
}
VLOG(2) << "(logid=" << log_id << ") key: " << keys[cube_val_idx] << " , cube value len:" << cur_val->buff.size();
memcpy(data_ptr, cur_val->buff.data()+10, cur_val->buff.size()-10);
memcpy(data_ptr, cur_val->buff.data(), cur_val->buff.size());
//VLOG(3) << keys[cube_val_idx] << ":" << data_ptr[0] << ", " << data_ptr[1] << ", " <<data_ptr[2] << ", " <<data_ptr[3] << ", " <<data_ptr[4] << ", " <<data_ptr[5] << ", " <<data_ptr[6] << ", " <<data_ptr[7] << ", " <<data_ptr[8];
++cube_key_found;
++cube_val_idx;
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册