未验证 提交 9899cfef 编写于 作者: J Jin Hai 提交者: GitHub

#1475 add debug timing for map uids (#1513)

Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
上级 fc8a6e07
...@@ -741,6 +741,17 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t ...@@ -741,6 +741,17 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_); return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
} }
// map offsets to ids
void
MapUids(const std::vector<segment::doc_id_t>& uids, int64_t* labels, size_t num) {
for (int64_t i = 0; i < num; ++i) {
int64_t& offset = labels[i];
if (offset != -1) {
offset = uids[offset];
}
}
}
Status Status
ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels, ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels,
bool hybrid) { bool hybrid) {
...@@ -798,7 +809,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr ...@@ -798,7 +809,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
} }
} }
#endif #endif
TimeRecorder rc("ExecutionEngineImpl::Search"); TimeRecorder rc("ExecutionEngineImpl::Search float");
if (index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
...@@ -824,15 +835,9 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr ...@@ -824,15 +835,9 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
rc.RecordSection("search done"); rc.RecordSection("search done");
// map offsets to ids // map offsets to ids
const std::vector<segment::doc_id_t>& uids = index_->GetUids(); MapUids(index_->GetUids(), labels, n * k);
for (int64_t i = 0; i < n * k; i++) {
int64_t offset = labels[i];
if (offset != -1) {
labels[i] = uids[offset];
}
}
rc.RecordSection("map uids"); rc.RecordSection("map uids " + std::to_string(n * k));
if (hybrid) { if (hybrid) {
HybridUnset(); HybridUnset();
...@@ -847,7 +852,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr ...@@ -847,7 +852,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
Status Status
ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t nprobe, float* distances, ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t nprobe, float* distances,
int64_t* labels, bool hybrid) { int64_t* labels, bool hybrid) {
TimeRecorder rc("ExecutionEngineImpl::Search"); TimeRecorder rc("ExecutionEngineImpl::Search uint8");
if (index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
...@@ -873,15 +878,9 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n ...@@ -873,15 +878,9 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n
rc.RecordSection("search done"); rc.RecordSection("search done");
// map offsets to ids // map offsets to ids
const std::vector<segment::doc_id_t>& uids = index_->GetUids(); MapUids(index_->GetUids(), labels, n * k);
for (int64_t i = 0; i < n * k; i++) {
int64_t offset = labels[i];
if (offset != -1) {
labels[i] = uids[offset];
}
}
rc.RecordSection("map uids"); rc.RecordSection("map uids " + std::to_string(n * k));
if (hybrid) { if (hybrid) {
HybridUnset(); HybridUnset();
...@@ -896,7 +895,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n ...@@ -896,7 +895,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, int64_t n
Status Status
ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t k, int64_t nprobe, float* distances, ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t k, int64_t nprobe, float* distances,
int64_t* labels, bool hybrid) { int64_t* labels, bool hybrid) {
TimeRecorder rc("ExecutionEngineImpl::Search"); TimeRecorder rc("ExecutionEngineImpl::Search vector of ids");
if (index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
...@@ -961,16 +960,12 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t ...@@ -961,16 +960,12 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t
auto status = Status::OK(); auto status = Status::OK();
if (!offsets.empty()) { if (!offsets.empty()) {
status = index_->SearchById(offsets.size(), offsets.data(), distances, labels, conf); status = index_->SearchById(offsets.size(), offsets.data(), distances, labels, conf);
rc.RecordSection("search by id done"); rc.RecordSection("search done");
// map offsets to ids // map offsets to ids
for (int64_t i = 0; i < offsets.size() * k; i++) { MapUids(uids, labels, offsets.size() * k);
int64_t offset = labels[i];
if (offset != -1) { rc.RecordSection("map uids " + std::to_string(offsets.size() * k));
labels[i] = uids[offset];
}
}
rc.RecordSection("map uids");
} }
if (hybrid) { if (hybrid) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册