Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
734a481b
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
734a481b
编写于
9月 10, 2019
作者:
X
xj.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-538 1. update kdt unittest
Former-commit-id: 449ccd913b9825e4563a79c1475c899fa05e4061
上级
878b69c6
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
71 addition
and
66 deletion
+71
-66
cpp/src/core/include/knowhere/index/preprocessor/normalize.h
cpp/src/core/include/knowhere/index/preprocessor/normalize.h
+26
-26
cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h
...rc/core/include/knowhere/index/vector_index/cpu_kdt_rng.h
+2
-2
cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp
cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp
+31
-31
cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
+7
-7
cpp/src/core/test/test_kdt.cpp
cpp/src/core/test/test_kdt.cpp
+5
-0
未找到文件。
cpp/src/core/include/knowhere/index/preprocessor/normalize.h
浏览文件 @
734a481b
#pragma once
#include <memory>
#include "preprocessor.h"
namespace
zilliz
{
namespace
knowhere
{
class
NormalizePreprocessor
:
public
Preprocessor
{
public:
DatasetPtr
Preprocess
(
const
DatasetPtr
&
input
)
override
;
private:
void
Normalize
(
float
*
arr
,
int64_t
dimension
);
};
using
NormalizePreprocessorPtr
=
std
::
shared_ptr
<
NormalizePreprocessor
>
;
}
// namespace knowhere
}
// namespace zilliz
//
#pragma once
//
//
#include <memory>
//
#include "preprocessor.h"
//
//
//
namespace zilliz {
//
namespace knowhere {
//
//
class NormalizePreprocessor : public Preprocessor {
//
public:
//
DatasetPtr
//
Preprocess(const DatasetPtr &input) override;
//
//
private:
//
//
void
//
Normalize(float *arr, int64_t dimension);
//
};
//
//
//
using NormalizePreprocessorPtr = std::shared_ptr<NormalizePreprocessor>;
//
//
//
} // namespace knowhere
//
} // namespace zilliz
cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h
浏览文件 @
734a481b
...
...
@@ -27,8 +27,8 @@ class CPUKDTRNG : public VectorIndex {
Load
(
const
BinarySet
&
index_array
)
override
;
public:
PreprocessorPtr
BuildPreprocessor
(
const
DatasetPtr
&
dataset
,
const
Config
&
config
)
override
;
//
PreprocessorPtr
//
BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override;
int64_t
Count
()
override
;
int64_t
Dimension
()
override
;
...
...
cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp
浏览文件 @
734a481b
#include "knowhere/index/vector_index/definitions.h"
#include "knowhere/common/config.h"
#include "knowhere/index/preprocessor/normalize.h"
namespace
zilliz
{
namespace
knowhere
{
DatasetPtr
NormalizePreprocessor
::
Preprocess
(
const
DatasetPtr
&
dataset
)
{
//
//
#include "knowhere/index/vector_index/definitions.h"
//
#include "knowhere/common/config.h"
//
#include "knowhere/index/preprocessor/normalize.h"
//
//
//
namespace zilliz {
//
namespace knowhere {
//
//
DatasetPtr
//
NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) {
// // TODO: wrap dataset->tensor
// auto tensor = dataset->tensor()[0];
// auto p_data = (float *)tensor->raw_mutable_data();
...
...
@@ -19,24 +19,24 @@ NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) {
// for (auto i = 0; i < rows; ++i) {
// Normalize(&(p_data[i * dimension]), dimension);
// }
}
void
NormalizePreprocessor
::
Normalize
(
float
*
arr
,
int64_t
dimension
)
{
//
double vector_length = 0;
//
for (auto j = 0; j < dimension; j++) {
//
double val = arr[j];
//
vector_length += val * val;
//
}
//
vector_length = std::sqrt(vector_length);
//
if (vector_length < 1e-6) {
//
auto val = (float) (1.0 / std::sqrt((double) dimension));
//
for (int j = 0; j < dimension; j++) arr[j] = val;
//
} else {
//
for (int j = 0; j < dimension; j++) arr[j] = (float) (arr[j] / vector_length);
//
}
}
}
// namespace knowhere
}
// namespace zilliz
//
}
//
//
void
//
NormalizePreprocessor::Normalize(float *arr, int64_t dimension) {
//
double vector_length = 0;
//
for (auto j = 0; j < dimension; j++) {
//
double val = arr[j];
//
vector_length += val * val;
//
}
//
vector_length = std::sqrt(vector_length);
//
if (vector_length < 1e-6) {
//
auto val = (float) (1.0 / std::sqrt((double) dimension));
//
for (int j = 0; j < dimension; j++) arr[j] = val;
//
} else {
//
for (int j = 0; j < dimension; j++) arr[j] = (float) (arr[j] / vector_length);
//
}
//
}
//
//
} // namespace knowhere
//
} // namespace zilliz
cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
浏览文件 @
734a481b
...
...
@@ -9,7 +9,7 @@
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
#include "knowhere/index/vector_index/definitions.h"
#include "knowhere/index/preprocessor/normalize.h"
//
#include "knowhere/index/preprocessor/normalize.h"
#include "knowhere/index/vector_index/kdt_parameters.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/common/exception.h"
...
...
@@ -60,10 +60,10 @@ CPUKDTRNG::Load(const BinarySet &binary_set) {
index_ptr_
->
LoadIndexFromMemory
(
index_blobs
);
}
PreprocessorPtr
CPUKDTRNG
::
BuildPreprocessor
(
const
DatasetPtr
&
dataset
,
const
Config
&
config
)
{
return
std
::
make_shared
<
NormalizePreprocessor
>
();
}
//
PreprocessorPtr
//
CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) {
//
return std::make_shared<NormalizePreprocessor>();
//
}
IndexModelPtr
CPUKDTRNG
::
Train
(
const
DatasetPtr
&
origin
,
const
Config
&
train_config
)
{
...
...
@@ -72,7 +72,7 @@ CPUKDTRNG::Train(const DatasetPtr &origin, const Config &train_config) {
//if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine
// && preprocessor_) {
preprocessor_
->
Preprocess
(
dataset
);
//
preprocessor_->Preprocess(dataset);
//}
auto
vectorset
=
ConvertToVectorSet
(
dataset
);
...
...
@@ -90,7 +90,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) {
//if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine
// && preprocessor_) {
preprocessor_
->
Preprocess
(
dataset
);
//
preprocessor_->Preprocess(dataset);
//}
auto
vectorset
=
ConvertToVectorSet
(
dataset
);
...
...
cpp/src/core/test/test_kdt.cpp
浏览文件 @
734a481b
...
...
@@ -8,6 +8,7 @@
#include <iostream>
#include <sstream>
#include "knowhere/common/exception.h"
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
#include "knowhere/index/vector_index/definitions.h"
...
...
@@ -125,6 +126,10 @@ TEST_P(KDTTest, kdt_serialize) {
auto
result
=
new_index
->
Search
(
query_dataset
,
search_cfg
);
AssertAnns
(
result
,
nq
,
k
);
PrintResult
(
result
,
nq
,
k
);
ASSERT_EQ
(
new_index
->
Count
(),
nb
);
ASSERT_EQ
(
new_index
->
Dimension
(),
dim
);
ASSERT_THROW
({
new_index
->
Clone
();},
zilliz
::
knowhere
::
KnowhereException
);
ASSERT_NO_THROW
({
new_index
->
Seal
();});
{
int
fileno
=
0
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录