Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
e9b70634
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e9b70634
编写于
4月 22, 2019
作者:
G
groot
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add id mapper class
Former-commit-id: b12be12ea33f4b3f99a4f65b1156d7a86007dcc1
上级
d8f40536
变更
9
显示空白变更内容
内联
并排
Showing
9 changed file
with
191 addition
and
27 deletion
+191
-27
cpp/src/server/VecIdMapper.cpp
cpp/src/server/VecIdMapper.cpp
+69
-2
cpp/src/server/VecIdMapper.h
cpp/src/server/VecIdMapper.h
+56
-3
cpp/src/server/VecServiceHandler.cpp
cpp/src/server/VecServiceHandler.cpp
+20
-5
cpp/unittest/CMakeLists.txt
cpp/unittest/CMakeLists.txt
+1
-2
cpp/unittest/log/CMakeLists.txt
cpp/unittest/log/CMakeLists.txt
+0
-10
cpp/unittest/server/CMakeLists.txt
cpp/unittest/server/CMakeLists.txt
+6
-5
cpp/unittest/server/cache_test.cpp
cpp/unittest/server/cache_test.cpp
+0
-0
cpp/unittest/server/idmapper_test.cpp
cpp/unittest/server/idmapper_test.cpp
+39
-0
cpp/unittest/server/log_test.cpp
cpp/unittest/server/log_test.cpp
+0
-0
未找到文件。
cpp/src/server/VecIdMapper.cpp
浏览文件 @
e9b70634
...
@@ -6,16 +6,83 @@
...
@@ -6,16 +6,83 @@
#include "VecIdMapper.h"
#include "VecIdMapper.h"
#include "utils/Log.h"
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
vecwise
{
namespace
server
{
namespace
server
{
VecIdMapper
::
VecIdMapper
()
{
IVecIdMapper
*
IVecIdMapper
::
GetInstance
()
{
static
SimpleIdMapper
s_mapper
;
return
&
s_mapper
;
}
SimpleIdMapper
::
SimpleIdMapper
()
{
}
SimpleIdMapper
::~
SimpleIdMapper
()
{
}
ServerError
SimpleIdMapper
::
Put
(
INTEGER_ID
nid
,
const
std
::
string
&
sid
)
{
ids_
[
nid
]
=
sid
;
return
SERVER_SUCCESS
;
}
ServerError
SimpleIdMapper
::
Put
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
const
std
::
vector
<
std
::
string
>&
sid
)
{
return
Put
(
nid
.
data
(),
nid
.
size
(),
sid
);
}
ServerError
SimpleIdMapper
::
Put
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
const
std
::
vector
<
std
::
string
>&
sid
)
{
if
(
count
!=
sid
.
size
())
{
return
SERVER_INVALID_ARGUMENT
;
}
for
(
int64_t
i
=
0
;
i
<
count
;
i
++
)
{
ids_
[
nid
[
i
]]
=
sid
[
i
];
}
return
SERVER_SUCCESS
;
}
ServerError
SimpleIdMapper
::
Get
(
INTEGER_ID
nid
,
std
::
string
&
sid
)
const
{
auto
iter
=
ids_
.
find
(
nid
);
if
(
iter
==
ids_
.
end
())
{
return
SERVER_INVALID_ARGUMENT
;
}
sid
=
iter
->
second
;
return
SERVER_SUCCESS
;
}
}
VecIdMapper
::~
VecIdMapper
()
{
ServerError
SimpleIdMapper
::
Get
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
std
::
vector
<
std
::
string
>&
sid
)
const
{
return
Get
(
nid
.
data
(),
nid
.
size
(),
sid
);
}
ServerError
SimpleIdMapper
::
Get
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
std
::
vector
<
std
::
string
>&
sid
)
const
{
sid
.
clear
();
ServerError
err
=
SERVER_SUCCESS
;
for
(
uint64_t
i
=
0
;
i
<
count
;
i
++
)
{
auto
iter
=
ids_
.
find
(
nid
[
i
]);
if
(
iter
==
ids_
.
end
())
{
sid
.
push_back
(
""
);
SERVER_LOG_ERROR
<<
"Failed to find id: "
<<
nid
[
i
];
err
=
SERVER_INVALID_ARGUMENT
;
continue
;
}
sid
.
push_back
(
iter
->
second
);
}
return
err
;
}
ServerError
SimpleIdMapper
::
Delete
(
INTEGER_ID
nid
)
{
ids_
.
erase
(
nid
);
return
SERVER_SUCCESS
;
}
}
}
}
...
...
cpp/src/server/VecIdMapper.h
浏览文件 @
e9b70634
...
@@ -5,18 +5,71 @@
...
@@ -5,18 +5,71 @@
******************************************************************************/
******************************************************************************/
#pragma once
#pragma once
#include "utils/Error.h"
#include <string>
#include <vector>
#include <unordered_map>
namespace
zilliz
{
namespace
zilliz
{
namespace
vecwise
{
namespace
vecwise
{
namespace
server
{
namespace
server
{
class
VecIdMapper
{
using
INTEGER_ID
=
int64_t
;
class
IVecIdMapper
{
public:
public:
VecIdMapper
();
static
IVecIdMapper
*
GetInstance
();
~
VecIdMapper
();
virtual
~
IVecIdMapper
(){}
virtual
ServerError
Put
(
INTEGER_ID
nid
,
const
std
::
string
&
sid
)
=
0
;
virtual
ServerError
Put
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
const
std
::
vector
<
std
::
string
>&
sid
)
=
0
;
virtual
ServerError
Put
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
const
std
::
vector
<
std
::
string
>&
sid
)
=
0
;
virtual
ServerError
Get
(
INTEGER_ID
nid
,
std
::
string
&
sid
)
const
=
0
;
virtual
ServerError
Get
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
std
::
vector
<
std
::
string
>&
sid
)
const
=
0
;
virtual
ServerError
Get
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
std
::
vector
<
std
::
string
>&
sid
)
const
=
0
;
virtual
ServerError
Delete
(
INTEGER_ID
nid
)
=
0
;
};
};
class
SimpleIdMapper
:
public
IVecIdMapper
{
public:
SimpleIdMapper
();
~
SimpleIdMapper
();
ServerError
Put
(
INTEGER_ID
nid
,
const
std
::
string
&
sid
)
override
;
ServerError
Put
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
const
std
::
vector
<
std
::
string
>&
sid
)
override
;
ServerError
Put
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
const
std
::
vector
<
std
::
string
>&
sid
)
override
;
ServerError
Get
(
INTEGER_ID
nid
,
std
::
string
&
sid
)
const
override
;
ServerError
Get
(
const
std
::
vector
<
INTEGER_ID
>&
nid
,
std
::
vector
<
std
::
string
>&
sid
)
const
override
;
ServerError
Get
(
const
INTEGER_ID
*
nid
,
uint64_t
count
,
std
::
vector
<
std
::
string
>&
sid
)
const
override
;
ServerError
Delete
(
INTEGER_ID
nid
)
override
;
private:
std
::
unordered_map
<
INTEGER_ID
,
std
::
string
>
ids_
;
};
//class RocksIdMapper : public IVecIdMapper{
//public:
// RocksIdMapper();
// ~RocksIdMapper();
//
// ServerError Put(INTEGER_ID nid, const std::string& sid) override;
// ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) override;
// ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) override;
//
// ServerError Get(INTEGER_ID nid, std::string& sid) const override;
// ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const override;
// ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const override;
//
// ServerError Delete(INTEGER_ID nid) override;
//
//};
}
}
}
}
}
}
cpp/src/server/VecServiceHandler.cpp
浏览文件 @
e9b70634
...
@@ -5,6 +5,7 @@
...
@@ -5,6 +5,7 @@
******************************************************************************/
******************************************************************************/
#include "VecServiceHandler.h"
#include "VecServiceHandler.h"
#include "ServerConfig.h"
#include "ServerConfig.h"
#include "VecIdMapper.h"
#include "utils/Log.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "utils/CommonUtil.h"
...
@@ -93,6 +94,11 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens
...
@@ -93,6 +94,11 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens
if
(
!
stat
.
ok
())
{
if
(
!
stat
.
ok
())
{
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
}
else
{
}
else
{
if
(
vector_ids
.
size
()
!=
1
)
{
SERVER_LOG_ERROR
<<
"Vector ID not returned"
;
}
else
{
IVecIdMapper
::
GetInstance
()
->
Put
(
vector_ids
[
0
],
tensor
.
uid
);
}
}
}
SERVER_LOG_INFO
<<
"add_vector() finished"
;
SERVER_LOG_INFO
<<
"add_vector() finished"
;
...
@@ -119,6 +125,13 @@ VecServiceHandler::add_vector_batch(const std::string &group_id,
...
@@ -119,6 +125,13 @@ VecServiceHandler::add_vector_batch(const std::string &group_id,
if
(
!
stat
.
ok
())
{
if
(
!
stat
.
ok
())
{
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
}
else
{
}
else
{
if
(
vector_ids
.
size
()
!=
tensor_list
.
tensor_list
.
size
())
{
SERVER_LOG_ERROR
<<
"Vector ID not returned"
;
}
else
{
for
(
size_t
i
=
0
;
i
<
vector_ids
.
size
();
i
++
)
{
IVecIdMapper
::
GetInstance
()
->
Put
(
vector_ids
[
i
],
tensor_list
.
tensor_list
[
i
].
uid
);
}
}
}
}
SERVER_LOG_INFO
<<
"add_vector_batch() finished"
;
SERVER_LOG_INFO
<<
"add_vector_batch() finished"
;
...
@@ -148,7 +161,9 @@ VecServiceHandler::search_vector(VecSearchResult &_return,
...
@@ -148,7 +161,9 @@ VecServiceHandler::search_vector(VecSearchResult &_return,
}
else
{
}
else
{
if
(
!
results
.
empty
())
{
if
(
!
results
.
empty
())
{
for
(
auto
id
:
results
[
0
])
{
for
(
auto
id
:
results
[
0
])
{
_return
.
id_list
.
push_back
(
std
::
to_string
(
id
));
std
::
string
sid
;
IVecIdMapper
::
GetInstance
()
->
Get
(
id
,
sid
);
_return
.
id_list
.
push_back
(
sid
);
}
}
}
}
}
}
...
@@ -182,13 +197,13 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return,
...
@@ -182,13 +197,13 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return,
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
SERVER_LOG_ERROR
<<
"Engine failed: "
<<
stat
.
ToString
();
}
else
{
}
else
{
for
(
engine
::
QueryResult
&
res
:
results
){
for
(
engine
::
QueryResult
&
res
:
results
){
for
(
auto
nid
:
results
)
{
VecSearchResult
v_res
;
VecSearchResult
v_res
;
for
(
auto
id
:
results
[
0
])
{
IVecIdMapper
::
GetInstance
()
->
Get
(
nid
,
v_res
.
id_list
);
v_res
.
id_list
.
push_back
(
std
::
to_string
(
id
));
}
_return
.
result_list
.
push_back
(
v_res
);
_return
.
result_list
.
push_back
(
v_res
);
}
}
}
}
}
SERVER_LOG_INFO
<<
"search_vector_batch() finished"
;
SERVER_LOG_INFO
<<
"search_vector_batch() finished"
;
}
catch
(
std
::
exception
&
ex
)
{
}
catch
(
std
::
exception
&
ex
)
{
...
...
cpp/unittest/CMakeLists.txt
浏览文件 @
e9b70634
...
@@ -16,7 +16,6 @@ set(unittest_libs
...
@@ -16,7 +16,6 @@ set(unittest_libs
gmock_main
gmock_main
pthread
)
pthread
)
add_subdirectory
(
cache
)
add_subdirectory
(
server
)
add_subdirectory
(
log
)
add_subdirectory
(
db
)
add_subdirectory
(
db
)
add_subdirectory
(
faiss_wrapper
)
add_subdirectory
(
faiss_wrapper
)
\ No newline at end of file
cpp/unittest/log/CMakeLists.txt
已删除
100644 → 0
浏览文件 @
d8f40536
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
set
(
log_test_src
${
unittest_srcs
}
log_test.cpp
)
add_executable
(
log_test
${
log_test_src
}
)
target_link_libraries
(
log_test
${
unittest_libs
}
)
cpp/unittest/
cache
/CMakeLists.txt
→
cpp/unittest/
server
/CMakeLists.txt
浏览文件 @
e9b70634
...
@@ -6,12 +6,13 @@
...
@@ -6,12 +6,13 @@
include_directories
(
../../src
)
include_directories
(
../../src
)
aux_source_directory
(
../../src/cache cache_srcs
)
aux_source_directory
(
../../src/cache cache_srcs
)
aux_source_directory
(
./ test_srcs
)
set
(
cache_test_src
add_executable
(
server_test
${
unittest_srcs
}
${
unittest_srcs
}
${
cache_srcs
}
${
cache_srcs
}
cache_test.cpp
)
${
test_srcs
}
../../src/server/VecIdMapper.cpp
)
add_executable
(
cache_test
${
cache_test_src
}
)
target_link_libraries
(
server_test
${
unittest_libs
}
)
target_link_libraries
(
cache_test
${
unittest_libs
}
)
cpp/unittest/
cache
/cache_test.cpp
→
cpp/unittest/
server
/cache_test.cpp
浏览文件 @
e9b70634
文件已移动
cpp/unittest/server/idmapper_test.cpp
0 → 100644
浏览文件 @
e9b70634
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "server/VecIdMapper.h"
using
namespace
zilliz
::
vecwise
;
TEST
(
IdMapperTest
,
IDMAPPER_TEST
)
{
server
::
IVecIdMapper
*
mapper
=
server
::
IVecIdMapper
::
GetInstance
();
std
::
vector
<
int64_t
>
nid
=
{
1
,
50
,
900
,
10000
};
std
::
vector
<
std
::
string
>
sid
=
{
"one"
,
"fifty"
,
"nine zero zero"
,
"many"
};
server
::
ServerError
err
=
mapper
->
Put
(
nid
,
sid
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
sid
.
clear
();
err
=
mapper
->
Put
(
nid
,
sid
);
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
std
::
vector
<
std
::
string
>
res
;
err
=
mapper
->
Get
(
nid
,
res
);
ASSERT_EQ
(
res
.
size
(),
nid
.
size
());
std
::
string
str_id
;
err
=
mapper
->
Get
(
50
,
str_id
);
ASSERT_EQ
(
str_id
,
"fifty"
);
err
=
mapper
->
Delete
(
900
);
ASSERT_EQ
(
err
,
server
::
SERVER_SUCCESS
);
err
=
mapper
->
Get
(
900
,
str_id
);
ASSERT_NE
(
err
,
server
::
SERVER_SUCCESS
);
}
cpp/unittest/
log
/log_test.cpp
→
cpp/unittest/
server
/log_test.cpp
浏览文件 @
e9b70634
文件已移动
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录