Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
4f5f9e17
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4f5f9e17
编写于
7月 08, 2019
作者:
X
xj.lin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update...
Former-commit-id: b99796b6e5fc88050b42c5ee295e66b4d111eab3
上级
e476273f
变更
20
显示空白变更内容
内联
并排
Showing
20 changed file
with
230 addition
and
87 deletion
+230
-87
cpp/src/CMakeLists.txt
cpp/src/CMakeLists.txt
+1
-1
cpp/src/cache/DataObj.h
cpp/src/cache/DataObj.h
+1
-1
cpp/src/db/EngineFactory.cpp
cpp/src/db/EngineFactory.cpp
+1
-1
cpp/src/db/ExecutionEngineImpl.cpp
cpp/src/db/ExecutionEngineImpl.cpp
+89
-32
cpp/src/db/ExecutionEngineImpl.h
cpp/src/db/ExecutionEngineImpl.h
+19
-9
cpp/src/db/FaissExecutionEngine.cpp
cpp/src/db/FaissExecutionEngine.cpp
+2
-0
cpp/src/db/FaissExecutionEngine.h
cpp/src/db/FaissExecutionEngine.h
+2
-0
cpp/src/wrapper/Index.cpp
cpp/src/wrapper/Index.cpp
+2
-0
cpp/src/wrapper/Index.h
cpp/src/wrapper/Index.h
+17
-10
cpp/src/wrapper/IndexBuilder.cpp
cpp/src/wrapper/IndexBuilder.cpp
+2
-2
cpp/src/wrapper/IndexBuilder.h
cpp/src/wrapper/IndexBuilder.h
+2
-0
cpp/src/wrapper/Operand.cpp
cpp/src/wrapper/Operand.cpp
+2
-0
cpp/src/wrapper/Operand.h
cpp/src/wrapper/Operand.h
+2
-0
cpp/src/wrapper/knowhere/data_transfer.cpp
cpp/src/wrapper/knowhere/data_transfer.cpp
+1
-1
cpp/src/wrapper/knowhere/data_transfer.h
cpp/src/wrapper/knowhere/data_transfer.h
+1
-1
cpp/src/wrapper/knowhere/vec_impl.cpp
cpp/src/wrapper/knowhere/vec_impl.cpp
+27
-10
cpp/src/wrapper/knowhere/vec_impl.h
cpp/src/wrapper/knowhere/vec_impl.h
+13
-2
cpp/src/wrapper/knowhere/vec_index.cpp
cpp/src/wrapper/knowhere/vec_index.cpp
+29
-13
cpp/src/wrapper/knowhere/vec_index.h
cpp/src/wrapper/knowhere/vec_index.h
+16
-3
cpp/thirdparty/knowhere
cpp/thirdparty/knowhere
+1
-1
未找到文件。
cpp/src/CMakeLists.txt
浏览文件 @
4f5f9e17
...
@@ -10,7 +10,7 @@ aux_source_directory(config config_files)
...
@@ -10,7 +10,7 @@ aux_source_directory(config config_files)
aux_source_directory
(
server server_files
)
aux_source_directory
(
server server_files
)
aux_source_directory
(
utils utils_files
)
aux_source_directory
(
utils utils_files
)
aux_source_directory
(
db db_files
)
aux_source_directory
(
db db_files
)
aux_source_directory
(
wrapper wrapper_files
)
#
aux_source_directory(wrapper wrapper_files)
aux_source_directory
(
metrics metrics_files
)
aux_source_directory
(
metrics metrics_files
)
aux_source_directory
(
wrapper/knowhere knowhere_files
)
aux_source_directory
(
wrapper/knowhere knowhere_files
)
...
...
cpp/src/cache/DataObj.h
浏览文件 @
4f5f9e17
...
@@ -28,7 +28,7 @@ public:
...
@@ -28,7 +28,7 @@ public:
return
0
;
return
0
;
}
}
return
index_
->
ntotal
*
(
index_
->
dim
*
4
);
return
index_
->
Count
()
*
index_
->
Dimension
()
*
sizeof
(
float
);
}
}
private:
private:
...
...
cpp/src/db/EngineFactory.cpp
浏览文件 @
4f5f9e17
...
@@ -4,7 +4,7 @@
...
@@ -4,7 +4,7 @@
* Proprietary and confidential.
* Proprietary and confidential.
******************************************************************************/
******************************************************************************/
#include "EngineFactory.h"
#include "EngineFactory.h"
#include "FaissExecutionEngine.h"
//
#include "FaissExecutionEngine.h"
#include "ExecutionEngineImpl.h"
#include "ExecutionEngineImpl.h"
#include "Log.h"
#include "Log.h"
...
...
cpp/src/db/ExecutionEngineImpl.cpp
浏览文件 @
4f5f9e17
...
@@ -3,13 +3,14 @@
...
@@ -3,13 +3,14 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
* Proprietary and confidential.
******************************************************************************/
******************************************************************************/
#include
"ExecutionEngineImpl.h"
#include
<src/server/ServerConfig.h>
#include "Log.h"
#include "Log.h"
#include "src/cache/CpuCacheMgr.h"
#include "ExecutionEngineImpl.h"
#include "wrapper/knowhere/vec_index.h"
#include "wrapper/knowhere/vec_impl.h"
#include "wrapper/knowhere/vec_impl.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
namespace
zilliz
{
namespace
zilliz
{
namespace
milvus
{
namespace
milvus
{
...
@@ -17,78 +18,119 @@ namespace engine {
...
@@ -17,78 +18,119 @@ namespace engine {
ExecutionEngineImpl
::
ExecutionEngineImpl
(
uint16_t
dimension
,
ExecutionEngineImpl
::
ExecutionEngineImpl
(
uint16_t
dimension
,
const
std
::
string
&
location
,
const
std
::
string
&
location
,
EngineType
type
)
EngineType
type
)
:
location_
(
location
)
{
:
location_
(
location
),
dim
(
dimension
),
build_type
(
type
)
{
index_
=
CreatetVecIndex
(
type
);
index_
=
CreatetVecIndex
(
EngineType
::
FAISS_IDMAP
);
std
::
static_pointer_cast
<
BFIndex
>
(
index_
)
->
Build
(
dimension
);
}
}
vecwise
::
engine
::
VecIndexPtr
ExecutionEngineImpl
::
CreatetVecIndex
(
EngineType
type
)
{
ExecutionEngineImpl
::
ExecutionEngineImpl
(
VecIndexPtr
index
,
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
;
const
std
::
string
&
location
,
switch
(
type
)
{
EngineType
type
)
case
EngineType
::
FAISS_IDMAP
:
{
:
index_
(
std
::
move
(
index
)),
location_
(
location
),
build_type
(
type
)
{
}
VecIndexPtr
ExecutionEngineImpl
::
CreatetVecIndex
(
EngineType
type
)
{
std
::
shared_ptr
<
VecIndex
>
index
;
switch
(
type
)
{
case
EngineType
::
FAISS_IDMAP
:
{
index
=
GetVecIndexFactory
(
IndexType
::
FAISS_IDMAP
);
break
;
break
;
}
}
case
EngineType
::
FAISS_IVFFLAT_GPU
:
{
case
EngineType
::
FAISS_IVFFLAT_GPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVF
>
(
0
);
index
=
GetVecIndexFactory
(
IndexType
::
FAISS_IVFFLAT_GPU
);
break
;
break
;
}
}
case
EngineType
::
FAISS_IVFFLAT_CPU
:
{
case
EngineType
::
FAISS_IVFFLAT_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
IVF
>
(
);
index
=
GetVecIndexFactory
(
IndexType
::
FAISS_IVFFLAT_CPU
);
break
;
break
;
}
}
case
EngineType
::
SPTAG_KDT_RNT_CPU
:
{
case
EngineType
::
SPTAG_KDT_RNT_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
CPUKDTRNG
>
(
);
index
=
GetVecIndexFactory
(
IndexType
::
SPTAG_KDT_RNT_CPU
);
break
;
break
;
}
}
default:
{
default:
{
ENGINE_LOG_ERROR
<<
"Invalid engine type"
;
ENGINE_LOG_ERROR
<<
"Invalid engine type"
;
return
nullptr
;
return
nullptr
;
}
}
}
}
return
index
;
return
std
::
make_shared
<
vecwise
::
engine
::
VecIndexImpl
>
(
index
);
}
}
Status
ExecutionEngineImpl
::
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
{
Status
ExecutionEngineImpl
::
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
{
index_
->
Add
(
n
,
xdata
,
xids
,
Config
::
object
{{
"dim"
,
dim
}});
return
Status
::
OK
();
return
Status
::
OK
();
}
}
size_t
ExecutionEngineImpl
::
Count
()
const
{
size_t
ExecutionEngineImpl
::
Count
()
const
{
return
0
;
return
index_
->
Count
()
;
}
}
size_t
ExecutionEngineImpl
::
Size
()
const
{
size_t
ExecutionEngineImpl
::
Size
()
const
{
return
0
;
return
(
size_t
)
(
Count
()
*
Dimension
())
*
sizeof
(
float
)
;
}
}
size_t
ExecutionEngineImpl
::
Dimension
()
const
{
size_t
ExecutionEngineImpl
::
Dimension
()
const
{
return
0
;
return
index_
->
Dimension
()
;
}
}
size_t
ExecutionEngineImpl
::
PhysicalSize
()
const
{
size_t
ExecutionEngineImpl
::
PhysicalSize
()
const
{
return
0
;
return
(
size_t
)
(
Count
()
*
Dimension
())
*
sizeof
(
float
)
;
}
}
Status
ExecutionEngineImpl
::
Serialize
()
{
Status
ExecutionEngineImpl
::
Serialize
()
{
// TODO(groot):
auto
binaryset
=
index_
->
Serialize
();
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
ExecutionEngineImpl
::
Load
()
{
Status
ExecutionEngineImpl
::
Load
()
{
// TODO(groot):
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
ExecutionEngineImpl
::
Merge
(
const
std
::
string
&
location
)
{
VecIndexPtr
ExecutionEngineImpl
::
Load
(
const
std
::
string
&
location
)
{
// TODO(groot): dev func in Fake code
// pseude code
//auto data = read_file(location);
//auto index_type = get_index_type(data);
//auto binaryset = get_index_binary(data);
/////
//return LoadVecIndex(index_type, binaryset);
return
nullptr
;
}
Status
ExecutionEngineImpl
::
Merge
(
const
std
::
string
&
location
)
{
if
(
location
==
location_
)
{
return
Status
::
Error
(
"Cannot Merge Self"
);
}
ENGINE_LOG_DEBUG
<<
"Merge index file: "
<<
location
<<
" to: "
<<
location_
;
auto
to_merge
=
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location
);
if
(
!
to_merge
)
{
to_merge
=
Load
(
location
);
}
auto
file_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
index_
);
index_
->
Add
(
file_index
->
Count
(),
file_index
->
GetRawVectors
(),
file_index
->
GetRawIds
());
return
Status
::
OK
();
return
Status
::
OK
();
}
}
// TODO(linxj): add config
ExecutionEnginePtr
ExecutionEnginePtr
ExecutionEngineImpl
::
BuildIndex
(
const
std
::
string
&
location
)
{
ExecutionEngineImpl
::
BuildIndex
(
const
std
::
string
&
location
)
{
return
nullptr
;
ENGINE_LOG_DEBUG
<<
"Build index file: "
<<
location
<<
" from: "
<<
location_
;
auto
from_index
=
std
::
dynamic_pointer_cast
<
BFIndex
>
(
index_
);
auto
to_index
=
CreatetVecIndex
(
build_type
);
to_index
->
BuildAll
(
Count
(),
from_index
->
GetRawVectors
(),
from_index
->
GetRawIds
(),
Config
::
object
{{
"dim"
,
Dimension
()},
{
"gpu_id"
,
gpu_num
}});
return
std
::
make_shared
<
ExecutionEngineImpl
>
(
to_index
,
location
,
build_type
);
}
}
Status
ExecutionEngineImpl
::
Search
(
long
n
,
Status
ExecutionEngineImpl
::
Search
(
long
n
,
...
@@ -96,16 +138,31 @@ Status ExecutionEngineImpl::Search(long n,
...
@@ -96,16 +138,31 @@ Status ExecutionEngineImpl::Search(long n,
long
k
,
long
k
,
float
*
distances
,
float
*
distances
,
long
*
labels
)
const
{
long
*
labels
)
const
{
index_
->
Search
(
n
,
data
,
distances
,
labels
,
Config
::
object
{{
"k"
,
k
},
{
"nprobe"
,
nprobe_
}});
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
ExecutionEngineImpl
::
Cache
()
{
Status
ExecutionEngineImpl
::
Cache
()
{
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
InsertItem
(
location_
,
index_
);
return
Status
::
OK
();
return
Status
::
OK
();
}
}
Status
ExecutionEngineImpl
::
Init
()
{
Status
ExecutionEngineImpl
::
Init
()
{
using
namespace
zilliz
::
milvus
::
server
;
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
server_config
=
config
.
GetConfig
(
CONFIG_SERVER
);
gpu_num
=
server_config
.
GetInt32Value
(
"gpu_index"
,
0
);
switch
(
build_type
)
{
case
EngineType
::
FAISS_IVFFLAT_GPU
:
{
}
case
EngineType
::
FAISS_IVFFLAT_CPU
:
{
ConfigNode
engine_config
=
config
.
GetConfig
(
CONFIG_ENGINE
);
nprobe_
=
engine_config
.
GetInt32Value
(
CONFIG_NPROBE
,
1000
);
break
;
}
}
return
Status
::
OK
();
return
Status
::
OK
();
}
}
...
...
cpp/src/db/ExecutionEngineImpl.h
浏览文件 @
4f5f9e17
...
@@ -11,16 +11,21 @@
...
@@ -11,16 +11,21 @@
#include <memory>
#include <memory>
#include <string>
#include <string>
namespace
zilliz
{
namespace
zilliz
{
namespace
milvus
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
class
ExecutionEngineImpl
:
public
ExecutionEngine
{
class
ExecutionEngineImpl
:
public
ExecutionEngine
{
public:
public:
ExecutionEngineImpl
(
uint16_t
dimension
,
ExecutionEngineImpl
(
uint16_t
dimension
,
const
std
::
string
&
location
,
const
std
::
string
&
location
,
EngineType
type
);
ExecutionEngineImpl
(
VecIndexPtr
index
,
const
std
::
string
&
location
,
EngineType
type
);
EngineType
type
);
Status
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
override
;
Status
AddWithIds
(
long
n
,
const
float
*
xdata
,
const
long
*
xids
)
override
;
...
@@ -37,7 +42,7 @@ public:
...
@@ -37,7 +42,7 @@ public:
Status
Load
()
override
;
Status
Load
()
override
;
Status
Merge
(
const
std
::
string
&
location
)
override
;
Status
Merge
(
const
std
::
string
&
location
)
override
;
Status
Search
(
long
n
,
Status
Search
(
long
n
,
const
float
*
data
,
const
float
*
data
,
...
@@ -45,21 +50,26 @@ public:
...
@@ -45,21 +50,26 @@ public:
float
*
distances
,
float
*
distances
,
long
*
labels
)
const
override
;
long
*
labels
)
const
override
;
ExecutionEnginePtr
BuildIndex
(
const
std
::
string
&
)
override
;
ExecutionEnginePtr
BuildIndex
(
const
std
::
string
&
)
override
;
Status
Cache
()
override
;
Status
Cache
()
override
;
Status
Init
()
override
;
Status
Init
()
override
;
private:
private:
vecwise
::
engine
::
VecIndexPtr
CreatetVecIndex
(
EngineType
type
);
VecIndexPtr
CreatetVecIndex
(
EngineType
type
);
VecIndexPtr
Load
(
const
std
::
string
&
location
);
protected:
protected:
vecwise
::
engine
::
VecIndexPtr
index_
;
VecIndexPtr
index_
=
nullptr
;
EngineType
build_type
;
int64_t
dim
;
std
::
string
location_
;
std
::
string
location_
;
size_t
nprobe_
=
0
;
size_t
nprobe_
=
0
;
int64_t
gpu_num
=
0
;
};
};
...
...
cpp/src/db/FaissExecutionEngine.cpp
浏览文件 @
4f5f9e17
...
@@ -3,6 +3,7 @@
...
@@ -3,6 +3,7 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
* Proprietary and confidential.
******************************************************************************/
******************************************************************************/
#if 0
#include "FaissExecutionEngine.h"
#include "FaissExecutionEngine.h"
#include "Log.h"
#include "Log.h"
...
@@ -181,3 +182,4 @@ Status FaissExecutionEngine::Init() {
...
@@ -181,3 +182,4 @@ Status FaissExecutionEngine::Init() {
} // namespace engine
} // namespace engine
} // namespace milvus
} // namespace milvus
} // namespace zilliz
} // namespace zilliz
#endif
cpp/src/db/FaissExecutionEngine.h
浏览文件 @
4f5f9e17
...
@@ -5,6 +5,7 @@
...
@@ -5,6 +5,7 @@
******************************************************************************/
******************************************************************************/
#pragma once
#pragma once
#if 0
#include "ExecutionEngine.h"
#include "ExecutionEngine.h"
#include "faiss/Index.h"
#include "faiss/Index.h"
...
@@ -71,3 +72,4 @@ protected:
...
@@ -71,3 +72,4 @@ protected:
} // namespace engine
} // namespace engine
} // namespace milvus
} // namespace milvus
} // namespace zilliz
} // namespace zilliz
#endif
cpp/src/wrapper/Index.cpp
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#if 0
// TODO: maybe support static search
// TODO: maybe support static search
#ifdef GPU_VERSION
#ifdef GPU_VERSION
#include "faiss/gpu/GpuAutoTune.h"
#include "faiss/gpu/GpuAutoTune.h"
...
@@ -80,3 +81,4 @@ Index_ptr read_index(const std::string &file_name) {
...
@@ -80,3 +81,4 @@ Index_ptr read_index(const std::string &file_name) {
}
}
}
}
}
}
#endif
cpp/src/wrapper/Index.h
浏览文件 @
4f5f9e17
...
@@ -6,23 +6,29 @@
...
@@ -6,23 +6,29 @@
#pragma once
#pragma once
#include <vector>
//#include <vector>
#include <string>
//#include <string>
#include <unordered_map>
//#include <unordered_map>
#include <memory>
//#include <memory>
#include <fstream>
//#include <fstream>
//
//#include "faiss/AutoTune.h"
//#include "faiss/index_io.h"
//
//#include "Operand.h"
#include "faiss/AutoTune.h"
#include "knowhere/vec_index.h"
#include "faiss/index_io.h"
#include "Operand.h"
namespace
zilliz
{
namespace
zilliz
{
namespace
milvus
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
class
Index
;
using
Index_ptr
=
VecIndexPtr
;
using
Index_ptr
=
std
::
shared_ptr
<
Index
>
;
#if 0
//class Index;
//using Index_ptr = std::shared_ptr<Index>;
class Index {
class Index {
typedef long idx_t;
typedef long idx_t;
...
@@ -75,6 +81,7 @@ private:
...
@@ -75,6 +81,7 @@ private:
void write_index(const Index_ptr &index, const std::string &file_name);
void write_index(const Index_ptr &index, const std::string &file_name);
extern Index_ptr read_index(const std::string &file_name);
extern Index_ptr read_index(const std::string &file_name);
#endif
}
}
...
...
cpp/src/wrapper/IndexBuilder.cpp
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#if 0
#include "mutex"
#include "mutex"
...
@@ -128,10 +129,8 @@ Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *i
...
@@ -128,10 +129,8 @@ Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *i
return
std
::
make_shared
<
Index
>
(
index
);
return
std
::
make_shared
<
Index
>
(
index
);
}
}
// TODO: Be Factory pattern later
IndexBuilderPtr
GetIndexBuilder
(
const
Operand_ptr
&
opd
)
{
IndexBuilderPtr
GetIndexBuilder
(
const
Operand_ptr
&
opd
)
{
if
(
opd
->
index_type
==
"IDMap"
)
{
if
(
opd
->
index_type
==
"IDMap"
)
{
// TODO: fix hardcode
IndexBuilderPtr
index
=
nullptr
;
IndexBuilderPtr
index
=
nullptr
;
return
std
::
make_shared
<
BgCpuBuilder
>
(
opd
);
return
std
::
make_shared
<
BgCpuBuilder
>
(
opd
);
}
}
...
@@ -142,3 +141,4 @@ IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd) {
...
@@ -142,3 +141,4 @@ IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd) {
}
}
}
}
}
}
#endif
cpp/src/wrapper/IndexBuilder.h
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#if 0
#pragma once
#pragma once
#include "faiss/Index.h"
#include "faiss/Index.h"
...
@@ -64,3 +65,4 @@ extern IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd);
...
@@ -64,3 +65,4 @@ extern IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd);
}
}
}
}
}
}
#endif
cpp/src/wrapper/Operand.cpp
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#if 0
#include "Operand.h"
#include "Operand.h"
...
@@ -90,3 +91,4 @@ Operand_ptr str_to_operand(const std::string &input) {
...
@@ -90,3 +91,4 @@ Operand_ptr str_to_operand(const std::string &input) {
}
}
}
}
}
}
#endif
cpp/src/wrapper/Operand.h
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#if 0
#pragma once
#pragma once
#include <string>
#include <string>
...
@@ -42,3 +43,4 @@ extern Operand_ptr str_to_operand(const std::string &input);
...
@@ -42,3 +43,4 @@ extern Operand_ptr str_to_operand(const std::string &input);
}
}
}
}
}
}
#endif
cpp/src/wrapper/knowhere/data_transfer.cpp
浏览文件 @
4f5f9e17
...
@@ -8,7 +8,7 @@
...
@@ -8,7 +8,7 @@
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
using
namespace
zilliz
::
knowhere
;
using
namespace
zilliz
::
knowhere
;
...
...
cpp/src/wrapper/knowhere/data_transfer.h
浏览文件 @
4f5f9e17
...
@@ -10,7 +10,7 @@
...
@@ -10,7 +10,7 @@
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
extern
zilliz
::
knowhere
::
DatasetPtr
extern
zilliz
::
knowhere
::
DatasetPtr
...
...
cpp/src/wrapper/knowhere/vec_impl.cpp
浏览文件 @
4f5f9e17
...
@@ -4,18 +4,14 @@
...
@@ -4,18 +4,14 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include "knowhere/index/index.h"
#include "knowhere/index/vector_index/idmap.h"
#include "knowhere/index/index_model.h"
#include "knowhere/index/index_type.h"
#include "knowhere/adapter/sptag.h"
#include "knowhere/common/tensor.h"
#include "vec_impl.h"
#include "vec_impl.h"
#include "data_transfer.h"
#include "data_transfer.h"
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
using
namespace
zilliz
::
knowhere
;
using
namespace
zilliz
::
knowhere
;
...
@@ -26,8 +22,8 @@ void VecIndexImpl::BuildAll(const long &nb,
...
@@ -26,8 +22,8 @@ void VecIndexImpl::BuildAll(const long &nb,
const
Config
&
cfg
,
const
Config
&
cfg
,
const
long
&
nt
,
const
long
&
nt
,
const
float
*
xt
)
{
const
float
*
xt
)
{
auto
d
=
cfg
[
"dim"
].
as
<
int
>
();
dim
=
cfg
[
"dim"
].
as
<
int
>
();
auto
dataset
=
GenDatasetWithIds
(
nb
,
d
,
xb
,
ids
);
auto
dataset
=
GenDatasetWithIds
(
nb
,
d
im
,
xb
,
ids
);
auto
preprocessor
=
index_
->
BuildPreprocessor
(
dataset
,
cfg
);
auto
preprocessor
=
index_
->
BuildPreprocessor
(
dataset
,
cfg
);
index_
->
set_preprocessor
(
preprocessor
);
index_
->
set_preprocessor
(
preprocessor
);
...
@@ -39,7 +35,7 @@ void VecIndexImpl::BuildAll(const long &nb,
...
@@ -39,7 +35,7 @@ void VecIndexImpl::BuildAll(const long &nb,
void
VecIndexImpl
::
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
{
void
VecIndexImpl
::
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
{
// TODO(linxj): Assert index is trained;
// TODO(linxj): Assert index is trained;
auto
d
=
cfg
[
"dim"
].
as
<
int
>
(
);
auto
d
=
cfg
.
get_with_default
(
"dim"
,
dim
);
auto
dataset
=
GenDatasetWithIds
(
nb
,
d
,
xb
,
ids
);
auto
dataset
=
GenDatasetWithIds
(
nb
,
d
,
xb
,
ids
);
index_
->
Add
(
dataset
,
cfg
);
index_
->
Add
(
dataset
,
cfg
);
...
@@ -48,8 +44,8 @@ void VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const C
...
@@ -48,8 +44,8 @@ void VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const C
void
VecIndexImpl
::
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
{
void
VecIndexImpl
::
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
{
// TODO: Assert index is trained;
// TODO: Assert index is trained;
auto
d
=
cfg
[
"dim"
].
as
<
int
>
();
auto
k
=
cfg
[
"k"
].
as
<
int
>
();
auto
k
=
cfg
[
"k"
].
as
<
int
>
();
auto
d
=
cfg
.
get_with_default
(
"dim"
,
dim
);
auto
dataset
=
GenDataset
(
nq
,
d
,
xq
);
auto
dataset
=
GenDataset
(
nq
,
d
,
xq
);
Config
search_cfg
;
Config
search_cfg
;
...
@@ -90,6 +86,27 @@ void VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) {
...
@@ -90,6 +86,27 @@ void VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) {
index_
->
Load
(
index_binary
);
index_
->
Load
(
index_binary
);
}
}
int64_t
VecIndexImpl
::
Dimension
()
{
return
index_
->
Dimension
();
}
int64_t
VecIndexImpl
::
Count
()
{
return
index_
->
Count
();
}
float
*
BFIndex
::
GetRawVectors
()
{
return
std
::
static_pointer_cast
<
IDMAP
>
(
index_
)
->
GetRawVectors
();
}
int64_t
*
BFIndex
::
GetRawIds
()
{
return
std
::
static_pointer_cast
<
IDMAP
>
(
index_
)
->
GetRawIds
();
}
void
BFIndex
::
Build
(
const
int64_t
&
d
)
{
dim
=
d
;
std
::
static_pointer_cast
<
IDMAP
>
(
index_
)
->
Train
(
dim
);
}
}
}
}
}
}
}
cpp/src/wrapper/knowhere/vec_impl.h
浏览文件 @
4f5f9e17
...
@@ -12,7 +12,7 @@
...
@@ -12,7 +12,7 @@
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
class
VecIndexImpl
:
public
VecIndex
{
class
VecIndexImpl
:
public
VecIndex
{
...
@@ -24,15 +24,26 @@ class VecIndexImpl : public VecIndex {
...
@@ -24,15 +24,26 @@ class VecIndexImpl : public VecIndex {
const
Config
&
cfg
,
const
Config
&
cfg
,
const
long
&
nt
,
const
long
&
nt
,
const
float
*
xt
)
override
;
const
float
*
xt
)
override
;
int64_t
Dimension
()
override
;
int64_t
Count
()
override
;
void
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
override
;
void
Add
(
const
long
&
nb
,
const
float
*
xb
,
const
long
*
ids
,
const
Config
&
cfg
)
override
;
zilliz
::
knowhere
::
BinarySet
Serialize
()
override
;
zilliz
::
knowhere
::
BinarySet
Serialize
()
override
;
void
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
void
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
override
;
void
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
override
;
void
Search
(
const
long
&
nq
,
const
float
*
xq
,
float
*
dist
,
long
*
ids
,
const
Config
&
cfg
)
override
;
private:
protected:
int64_t
dim
;
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index_
=
nullptr
;
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index_
=
nullptr
;
};
};
class
BFIndex
:
public
VecIndexImpl
{
public:
explicit
BFIndex
(
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
)
:
VecIndexImpl
(
std
::
move
(
index
))
{};
void
Build
(
const
int64_t
&
d
);
float
*
GetRawVectors
();
int64_t
*
GetRawIds
();
};
}
}
}
}
}
}
cpp/src/wrapper/knowhere/vec_index.cpp
浏览文件 @
4f5f9e17
...
@@ -4,6 +4,7 @@
...
@@ -4,6 +4,7 @@
// Proprietary and confidential.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/ivf.h"
#include "knowhere/index/vector_index/idmap.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
...
@@ -12,27 +13,42 @@
...
@@ -12,27 +13,42 @@
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
// TODO(linxj): index_type => enum struct
// TODO(linxj): index_type => enum struct
VecIndexPtr
GetVecIndexFactory
(
const
std
::
string
&
index_
type
)
{
VecIndexPtr
GetVecIndexFactory
(
const
IndexType
&
type
)
{
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
;
std
::
shared_ptr
<
zilliz
::
knowhere
::
VectorIndex
>
index
;
if
(
index_type
==
"IVF"
)
{
switch
(
type
)
{
case
IndexType
::
FAISS_IDMAP
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
IDMAP
>
();
return
std
::
make_shared
<
BFIndex
>
(
index
);
}
case
IndexType
::
FAISS_IVFFLAT_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
IVF
>
();
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
IVF
>
();
}
else
if
(
index_type
==
"GPUIVF"
)
{
break
;
}
case
IndexType
::
FAISS_IVFFLAT_GPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVF
>
(
0
);
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
GPUIVF
>
(
0
);
}
else
if
(
index_type
==
"SPTAG"
)
{
break
;
}
case
IndexType
::
SPTAG_KDT_RNT_CPU
:
{
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
CPUKDTRNG
>
();
index
=
std
::
make_shared
<
zilliz
::
knowhere
::
CPUKDTRNG
>
();
break
;
}
}
// TODO(linxj): Support NSG
//
// TODO(linxj): Support NSG
//else if (index_type == "NSG")
{
//case IndexType ::NSG:
{
// index = std::make_shared<zilliz::knowhere::NSG>();
// index = std::make_shared<zilliz::knowhere::NSG>();
// break;
//}
//}
default:
{
return
nullptr
;
}
}
return
std
::
make_shared
<
VecIndexImpl
>
(
index
);
return
std
::
make_shared
<
VecIndexImpl
>
(
index
);
}
}
VecIndexPtr
LoadVecIndex
(
const
std
::
string
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
VecIndexPtr
LoadVecIndex
(
const
IndexType
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
{
auto
index
=
GetVecIndexFactory
(
index_type
);
auto
index
=
GetVecIndexFactory
(
index_type
);
index
->
Load
(
index_binary
);
index
->
Load
(
index_binary
);
return
index
;
return
index
;
...
...
cpp/src/wrapper/knowhere/vec_index.h
浏览文件 @
4f5f9e17
...
@@ -14,7 +14,7 @@
...
@@ -14,7 +14,7 @@
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
milvus
{
namespace
engine
{
namespace
engine
{
// TODO(linxj): jsoncons => rapidjson or other.
// TODO(linxj): jsoncons => rapidjson or other.
...
@@ -40,6 +40,10 @@ class VecIndex {
...
@@ -40,6 +40,10 @@ class VecIndex {
long
*
ids
,
long
*
ids
,
const
Config
&
cfg
=
Config
())
=
0
;
const
Config
&
cfg
=
Config
())
=
0
;
virtual
int64_t
Dimension
()
=
0
;
virtual
int64_t
Count
()
=
0
;
virtual
zilliz
::
knowhere
::
BinarySet
Serialize
()
=
0
;
virtual
zilliz
::
knowhere
::
BinarySet
Serialize
()
=
0
;
virtual
void
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
=
0
;
virtual
void
Load
(
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
)
=
0
;
...
@@ -47,9 +51,18 @@ class VecIndex {
...
@@ -47,9 +51,18 @@ class VecIndex {
using
VecIndexPtr
=
std
::
shared_ptr
<
VecIndex
>
;
using
VecIndexPtr
=
std
::
shared_ptr
<
VecIndex
>
;
extern
VecIndexPtr
GetVecIndexFactory
(
const
std
::
string
&
index_type
);
enum
class
IndexType
{
INVALID
=
0
,
FAISS_IDMAP
=
1
,
FAISS_IVFFLAT_GPU
,
FAISS_IVFFLAT_CPU
,
SPTAG_KDT_RNT_CPU
,
NSG
,
};
extern
VecIndexPtr
GetVecIndexFactory
(
const
IndexType
&
type
);
extern
VecIndexPtr
LoadVecIndex
(
const
std
::
string
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
);
extern
VecIndexPtr
LoadVecIndex
(
const
IndexType
&
index_type
,
const
zilliz
::
knowhere
::
BinarySet
&
index_binary
);
}
}
}
}
...
...
knowhere
@
c0df7662
Subproject commit
2d543bfab655398f30113681f348519acac40ab5
Subproject commit
c0df766214d7fa288ffedd77cd06a8ba8620c8df
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录