Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
0d25f30b
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0d25f30b
编写于
7月 03, 2019
作者:
Y
yu yunfeng
提交者:
jinhai
7月 13, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update IVF nprobe
Former-commit-id: d2e93de87f9c8b45551f1453b5c0303508cab0c5
上级
6ae16689
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
65 addition
and
44 deletion
+65
-44
cpp/conf/server_config.template
cpp/conf/server_config.template
+4
-1
cpp/src/db/EngineFactory.cpp
cpp/src/db/EngineFactory.cpp
+25
-9
cpp/src/db/ExecutionEngine.h
cpp/src/db/ExecutionEngine.h
+2
-0
cpp/src/db/FaissExecutionEngine.cpp
cpp/src/db/FaissExecutionEngine.cpp
+29
-1
cpp/src/db/FaissExecutionEngine.h
cpp/src/db/FaissExecutionEngine.h
+5
-4
cpp/src/wrapper/Index.cpp
cpp/src/wrapper/Index.cpp
+0
-29
未找到文件。
cpp/conf/server_config.template
浏览文件 @
0d25f30b
...
...
@@ -30,4 +30,7 @@ license_config: # license configure
license_path: "@MILVUS_DB_PATH@/system.license" # license file path
cache_config: # cache configure
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
\ No newline at end of file
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
engine_config:
nprobe: 10
\ No newline at end of file
cpp/src/db/EngineFactory.cpp
浏览文件 @
0d25f30b
...
...
@@ -7,23 +7,39 @@
#include "FaissExecutionEngine.h"
#include "Log.h"
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
ExecutionEnginePtr
EngineFactory
::
Build
(
uint16_t
dimension
,
const
std
::
string
&
location
,
EngineType
type
)
{
switch
(
type
)
{
case
EngineType
::
FAISS_IDMAP
:
return
ExecutionEnginePtr
(
new
FaissExecutionEngine
(
dimension
,
location
,
"IDMap"
,
"IDMap,Flat"
));
case
EngineType
::
FAISS_IVFFLAT
:
return
ExecutionEnginePtr
(
new
FaissExecutionEngine
(
dimension
,
location
,
"IVF"
,
"IDMap,Flat"
));
default:
ENGINE_LOG_ERROR
<<
"Unsupportted engine type"
;
const
std
::
string
&
location
,
EngineType
type
)
{
ExecutionEnginePtr
execution_engine_ptr
;
switch
(
type
)
{
case
EngineType
::
FAISS_IDMAP
:
{
execution_engine_ptr
=
ExecutionEnginePtr
(
new
FaissExecutionEngine
(
dimension
,
location
,
"IDMap"
,
"IDMap,Flat"
));
break
;
}
case
EngineType
::
FAISS_IVFFLAT
:
{
execution_engine_ptr
=
ExecutionEnginePtr
(
new
FaissExecutionEngine
(
dimension
,
location
,
"IVF"
,
"IDMap,Flat"
));
break
;
}
default:
{
ENGINE_LOG_ERROR
<<
"Unsupported engine type"
;
return
nullptr
;
}
}
execution_engine_ptr
->
Init
();
return
execution_engine_ptr
;
}
}
...
...
cpp/src/db/ExecutionEngine.h
浏览文件 @
0d25f30b
...
...
@@ -50,6 +50,8 @@ public:
virtual
std
::
shared_ptr
<
ExecutionEngine
>
BuildIndex
(
const
std
::
string
&
)
=
0
;
virtual
Status
Cache
()
=
0
;
virtual
Status
Init
()
=
0
;
};
using
ExecutionEnginePtr
=
std
::
shared_ptr
<
ExecutionEngine
>
;
...
...
cpp/src/db/FaissExecutionEngine.cpp
浏览文件 @
0d25f30b
...
...
@@ -13,6 +13,7 @@
#include <wrapper/Index.h>
#include <wrapper/IndexBuilder.h>
#include <cache/CpuCacheMgr.h>
#include "faiss/IndexIVF.h"
#include "metrics/Metrics.h"
...
...
@@ -135,7 +136,16 @@ Status FaissExecutionEngine::Search(long n,
float
*
distances
,
long
*
labels
)
const
{
auto
start_time
=
METRICS_NOW_TIME
;
pIndex_
->
search
(
n
,
data
,
k
,
distances
,
labels
);
std
::
shared_ptr
<
faiss
::
IndexIVF
>
ivf_index
=
std
::
dynamic_pointer_cast
<
faiss
::
IndexIVF
>
(
pIndex_
);
if
(
ivf_index
)
{
ENGINE_LOG_DEBUG
<<
"Index type: IVFFLAT nProbe: "
<<
nprobe_
;
ivf_index
->
nprobe
=
nprobe_
;
ivf_index
->
search
(
n
,
data
,
k
,
distances
,
labels
);
}
else
{
pIndex_
->
search
(
n
,
data
,
k
,
distances
,
labels
);
}
auto
end_time
=
METRICS_NOW_TIME
;
auto
total_time
=
METRICS_MICROSECONDS
(
start_time
,
end_time
);
server
::
Metrics
::
GetInstance
().
QueryIndexTypePerSecondSet
(
build_index_type_
,
double
(
n
)
/
double
(
total_time
));
...
...
@@ -149,6 +159,24 @@ Status FaissExecutionEngine::Cache() {
return
Status
::
OK
();
}
Status
FaissExecutionEngine
::
Init
()
{
if
(
build_index_type_
==
"IVF"
)
{
using
namespace
zilliz
::
milvus
::
server
;
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
engine_config
=
config
.
GetConfig
(
CONFIG_ENGINE
);
nprobe_
=
engine_config
.
GetInt32Value
(
CONFIG_NPROBE
,
1000
);
}
else
if
(
build_index_type_
==
"IDMap"
)
{
;
}
else
{
return
Status
::
Error
(
"Wrong index type: "
,
build_index_type_
);
}
return
Status
::
OK
();
}
}
// namespace engine
}
// namespace milvus
...
...
cpp/src/db/FaissExecutionEngine.h
浏览文件 @
0d25f30b
...
...
@@ -6,14 +6,11 @@
#pragma once
#include "ExecutionEngine.h"
#include "faiss/Index.h"
#include <memory>
#include <string>
namespace
faiss
{
class
Index
;
}
namespace
zilliz
{
namespace
milvus
{
namespace
engine
{
...
...
@@ -58,12 +55,16 @@ public:
Status
Cache
()
override
;
Status
Init
()
override
;
protected:
std
::
shared_ptr
<
faiss
::
Index
>
pIndex_
;
std
::
string
location_
;
std
::
string
build_index_type_
;
std
::
string
raw_index_type_
;
size_t
nprobe_
=
0
;
};
...
...
cpp/src/wrapper/Index.cpp
浏览文件 @
0d25f30b
...
...
@@ -25,32 +25,6 @@ using std::string;
using
std
::
unordered_map
;
using
std
::
vector
;
class
Nprobe
{
public:
static
Nprobe
&
GetInstance
()
{
static
Nprobe
instance
;
return
instance
;
}
void
SelectNprobe
()
{
using
namespace
zilliz
::
milvus
::
server
;
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
engine_config
=
config
.
GetConfig
(
CONFIG_ENGINE
);
nprobe_
=
engine_config
.
GetInt32Value
(
CONFIG_NPROBE
,
1000
);
}
size_t
GetNprobe
()
{
return
nprobe_
;
}
private:
Nprobe
()
:
nprobe_
(
1000
)
{
SelectNprobe
();
}
private:
size_t
nprobe_
;
};
Index
::
Index
(
const
std
::
shared_ptr
<
faiss
::
Index
>
&
raw_index
)
{
index_
=
raw_index
;
dim
=
index_
->
d
;
...
...
@@ -84,9 +58,6 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) {
bool
Index
::
search
(
idx_t
n
,
const
float
*
data
,
idx_t
k
,
float
*
distances
,
long
*
labels
)
const
{
try
{
if
(
auto
ivf_index
=
std
::
dynamic_pointer_cast
<
faiss
::
IndexIVF
>
(
index_
))
{
ivf_index
->
nprobe
=
Nprobe
::
GetInstance
().
GetNprobe
();
}
index_
->
search
(
n
,
data
,
k
,
distances
,
labels
);
}
catch
(
std
::
exception
&
e
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录