Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
3f7fe254
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3f7fe254
编写于
4月 23, 2019
作者:
G
groot
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add id mapper
Former-commit-id: 56cfee9e3beeb474e2a56c3e3a3fdda111cfabb2
上级
8909b197
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
131 addition
and
15 deletion
+131
-15
cpp/README.md
cpp/README.md
+5
-2
cpp/src/CMakeLists.txt
cpp/src/CMakeLists.txt
+10
-2
cpp/src/server/VecIdMapper.cpp
cpp/src/server/VecIdMapper.cpp
+93
-7
cpp/src/server/VecIdMapper.h
cpp/src/server/VecIdMapper.h
+6
-0
cpp/src/server/VecServiceHandler.cpp
cpp/src/server/VecServiceHandler.cpp
+2
-0
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+1
-1
cpp/unittest/server/CMakeLists.txt
cpp/unittest/server/CMakeLists.txt
+10
-1
cpp/unittest/server/idmapper_test.cpp
cpp/unittest/server/idmapper_test.cpp
+4
-2
未找到文件。
cpp/README.md
浏览文件 @
3f7fe254
### Compilation
#### Step 1: install necessery tools
centos7 : yum install gfortran
ubuntu16.04 : sudo apt-install install gfortran libsqlite3-dev
centos7 :
yum install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
ubuntu16.04 :
sudo apt-install install gfortran libsqlite3-dev libsnappy-dev zlib bzip2
#### Step 2: build third-parties
Note: If you want to debug into third-parties, you can build debug with CXXFLAGS='-g -O0' with option
...
...
cpp/src/CMakeLists.txt
浏览文件 @
3f7fe254
...
...
@@ -24,6 +24,7 @@ set(vecwise_engine_src
${
wrapper_files
}
)
include_directories
(
/usr/include
)
include_directories
(
/usr/local/cuda/include
)
find_library
(
cuda_library cudart cublas HINTS /usr/local/cuda/lib64
)
...
...
@@ -37,6 +38,8 @@ set(engine_libs
sqlite3
)
link_libraries
(
/usr/lib/x86_64-linux-gnu
)
target_link_libraries
(
vecwise_engine
${
engine_libs
}
${
cuda_library
}
)
add_executable
(
vecwise_server
...
...
@@ -48,12 +51,17 @@ add_executable(vecwise_server
)
set
(
server_libs
vecwise_engine
rocksdb
thrift
pthread
yaml-cpp
boost_system
boost_filesystem
thrift
pthread
vecwise_engine
snappy
bz2
z
)
target_link_libraries
(
vecwise_server
${
server_libs
}
)
cpp/src/server/VecIdMapper.cpp
浏览文件 @
3f7fe254
...
...
@@ -7,19 +7,25 @@
#include "VecIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
namespace
zilliz
{
namespace
vecwise
{
namespace
server
{
IVecIdMapper
*
IVecIdMapper
::
GetInstance
()
{
#if
1
#if
0
static SimpleIdMapper s_mapper;
return &s_mapper;
#else
ConfigNode
&
config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_SERVER
);
std
::
string
db_path
=
config
.
GetValue
(
CONFIG_SERVER_DB_PATH
);
db_path
+=
"/id_mapping"
;
CommonUtil
::
CreateDirectory
(
db_path
);
static
RocksIdMapper
s_mapper
(
db_path
);
return
&
s_mapper
;
#endif
...
...
@@ -78,7 +84,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect
auto
iter
=
ids_
.
find
(
nid
[
i
]);
if
(
iter
==
ids_
.
end
())
{
sid
.
push_back
(
""
);
SERVER_LOG_ERROR
<<
"
F
ailed to find id: "
<<
nid
[
i
];
SERVER_LOG_ERROR
<<
"
ID mapper f
ailed to find id: "
<<
nid
[
i
];
err
=
SERVER_INVALID_ARGUMENT
;
continue
;
}
...
...
@@ -97,37 +103,117 @@ ServerError SimpleIdMapper::Delete(INTEGER_ID nid) {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
RocksIdMapper
::
RocksIdMapper
(
const
std
::
string
&
store_path
)
{
rocksdb
::
Options
options
;
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
options
.
IncreaseParallelism
();
options
.
OptimizeLevelStyleCompaction
();
// create the DB if it's not already present
options
.
create_if_missing
=
true
;
// open DB
rocksdb
::
Status
s
=
rocksdb
::
DB
::
Open
(
options
,
store_path
,
&
db_
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
"ID mapper failed to initialize:"
<<
s
.
ToString
();
db_
=
nullptr
;
}
}
RocksIdMapper
::~
RocksIdMapper
()
{
if
(
db_
)
{
db_
->
Close
();
delete
db_
;
}
}
ServerError
RocksIdMapper
::
Put
(
INTEGER_ID
nid
,
const
std
::
string
&
sid
)
{
if
(
db_
==
nullptr
)
{
return
SERVER_NULL_POINTER
;
}
std
::
string
str_id
=
std
::
to_string
(
nid
);
//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb
::
Slice
key
(
str_id
);
rocksdb
::
Slice
value
(
sid
);
rocksdb
::
Status
s
=
db_
->
Put
(
rocksdb
::
WriteOptions
(),
key
,
value
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
"ID mapper failed to put:"
<<
s
.
ToString
();
return
SERVER_UNEXPECTED_ERROR
;
}
return
SERVER_SUCCESS
;
}
ServerError
RocksIdMapper
::
Put
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
const
std
::
vector
<
std
::
string
>&
sid
)
{
return
SERVER_SUCCESS
;
return
Put
(
nid
.
data
(),
nid
.
size
(),
sid
)
;
}
ServerError
RocksIdMapper
::
Put
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
const
std
::
vector
<
std
::
string
>&
sid
)
{
if
(
count
!=
sid
.
size
())
{
return
SERVER_INVALID_ARGUMENT
;
}
ServerError
err
=
SERVER_SUCCESS
;
for
(
int64_t
i
=
0
;
i
<
count
;
i
++
)
{
err
=
Put
(
nid
[
i
],
sid
[
i
]);
if
(
err
!=
SERVER_SUCCESS
)
{
return
err
;
}
}
return
SERVER_SUCCESS
;
}
ServerError
RocksIdMapper
::
Get
(
INTEGER_ID
nid
,
std
::
string
&
sid
)
const
{
if
(
db_
==
nullptr
)
{
return
SERVER_NULL_POINTER
;
}
std
::
string
str_id
=
std
::
to_string
(
nid
);
//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb
::
Slice
key
(
str_id
);
rocksdb
::
Status
s
=
db_
->
Get
(
rocksdb
::
ReadOptions
(),
key
,
&
sid
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
"ID mapper failed to get:"
<<
s
.
ToString
();
return
SERVER_UNEXPECTED_ERROR
;
}
return
SERVER_SUCCESS
;
}
ServerError
RocksIdMapper
::
Get
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
std
::
vector
<
std
::
string
>&
sid
)
const
{
return
SERVER_SUCCESS
;
return
Get
(
nid
.
data
(),
nid
.
size
(),
sid
)
;
}
ServerError
RocksIdMapper
::
Get
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
std
::
vector
<
std
::
string
>&
sid
)
const
{
return
SERVER_SUCCESS
;
sid
.
clear
();
ServerError
err
=
SERVER_SUCCESS
;
for
(
uint64_t
i
=
0
;
i
<
count
;
i
++
)
{
std
::
string
str_id
;
ServerError
temp_err
=
Get
(
nid
[
i
],
str_id
);
if
(
temp_err
!=
SERVER_SUCCESS
)
{
sid
.
push_back
(
""
);
SERVER_LOG_ERROR
<<
"ID mapper failed to get id: "
<<
nid
[
i
];
err
=
temp_err
;
continue
;
}
sid
.
push_back
(
str_id
);
}
return
err
;
}
ServerError
RocksIdMapper
::
Delete
(
INTEGER_ID
nid
)
{
if
(
db_
==
nullptr
)
{
return
SERVER_NULL_POINTER
;
}
std
::
string
str_id
=
std
::
to_string
(
nid
);
//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb
::
Slice
key
(
str_id
);
rocksdb
::
Status
s
=
db_
->
Delete
(
rocksdb
::
WriteOptions
(),
key
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
"ID mapper failed to delete:"
<<
s
.
ToString
();
return
SERVER_UNEXPECTED_ERROR
;
}
return
SERVER_SUCCESS
;
}
...
...
cpp/src/server/VecIdMapper.h
浏览文件 @
3f7fe254
...
...
@@ -11,6 +11,10 @@
#include <vector>
#include <unordered_map>
namespace
rocksdb
{
class
DB
;
}
namespace
zilliz
{
namespace
vecwise
{
namespace
server
{
...
...
@@ -68,6 +72,8 @@ public:
ServerError
Delete
(
INTEGER_ID
nid
)
override
;
private:
rocksdb
::
DB
*
db_
;
};
}
...
...
cpp/src/server/VecServiceHandler.cpp
浏览文件 @
3f7fe254
...
...
@@ -23,6 +23,8 @@ VecServiceHandler::VecServiceHandler() {
std
::
string
db_path
=
config
.
GetValue
(
CONFIG_SERVER_DB_PATH
);
opt
.
meta
.
path
=
db_path
+
"/db"
;
CommonUtil
::
CreateDirectory
(
opt
.
meta
.
path
);
zilliz
::
vecwise
::
engine
::
DB
::
Open
(
opt
,
&
db_
);
}
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
3f7fe254
...
...
@@ -34,7 +34,7 @@ TEST(DBTest, DB_TEST) {
group_info
.
dimension
=
group_dim
;
group_info
.
group_id
=
group_name
;
engine
::
Status
stat
=
db
->
add_group
(
group_info
);
ASSERT_STATS
(
stat
);
//
ASSERT_STATS(stat);
engine
::
meta
::
GroupSchema
group_info_get
;
group_info_get
.
group_id
=
group_name
;
...
...
cpp/unittest/server/CMakeLists.txt
浏览文件 @
3f7fe254
...
...
@@ -4,6 +4,7 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
include_directories
(
../../src
)
include_directories
(
/usr/include
)
aux_source_directory
(
../../src/config config_files
)
aux_source_directory
(
../../src/cache cache_srcs
)
...
...
@@ -12,6 +13,7 @@ aux_source_directory(./ test_srcs)
set
(
require_files
../../src/server/VecIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
)
add_executable
(
server_test
...
...
@@ -26,7 +28,14 @@ set(require_libs
yaml-cpp
boost_system
boost_filesystem
pthread
)
pthread
snappy
bz2
z
rocksdb
)
link_libraries
(
/usr/lib/x86_64-linux-gnu
)
target_link_libraries
(
server_test
${
unittest_libs
}
...
...
cpp/unittest/server/idmapper_test.cpp
浏览文件 @
3f7fe254
...
...
@@ -21,13 +21,15 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
server
::
ServerError
err
=
mapper
->
Put
(
nid
,
sid
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
sid
.
clear
();
err
=
mapper
->
Put
(
nid
,
sid
);
err
=
mapper
->
Put
(
nid
,
std
::
vector
<
std
::
string
>
());
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
std
::
vector
<
std
::
string
>
res
;
err
=
mapper
->
Get
(
nid
,
res
);
ASSERT_EQ
(
res
.
size
(),
nid
.
size
());
for
(
size_t
i
=
0
;
i
<
res
.
size
();
i
++
)
{
ASSERT_EQ
(
res
[
i
],
sid
[
i
]);
}
std
::
string
str_id
;
err
=
mapper
->
Get
(
50
,
str_id
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录