Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
964d3f37
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
964d3f37
编写于
6月 10, 2019
作者:
Y
Yang Xuan
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'mega/branch-0.3.0' into branch-0.3.0
Former-commit-id: 8c8fcafbfd56d787db947bc82b12d89b33dc7ec7
上级
6f2c5885
1a0596b2
变更
7
显示空白变更内容
内联
并排
Showing
7 changed file
with
175 addition
and
76 deletion
+175
-76
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+1
-0
cpp/src/metrics/PrometheusMetrics.cpp
cpp/src/metrics/PrometheusMetrics.cpp
+18
-13
cpp/src/sdk/examples/simple/src/ClientTest.cpp
cpp/src/sdk/examples/simple/src/ClientTest.cpp
+16
-0
cpp/src/sdk/src/client/ClientProxy.cpp
cpp/src/sdk/src/client/ClientProxy.cpp
+13
-0
cpp/src/server/MegasearchTask.cpp
cpp/src/server/MegasearchTask.cpp
+86
-53
cpp/src/utils/CommonUtil.cpp
cpp/src/utils/CommonUtil.cpp
+34
-9
cpp/src/utils/CommonUtil.h
cpp/src/utils/CommonUtil.h
+7
-1
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
964d3f37
...
...
@@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-56 - Add version information when server is started
-
MS-64 - Different table can have different index type
-
MS-52 - Return search score
-
MS-66 - Support time range query
## Task
...
...
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @
964d3f37
...
...
@@ -5,7 +5,7 @@
******************************************************************************/
#include "PrometheusMetrics.h"
#include "utils/Log.h"
namespace
zilliz
{
namespace
vecwise
{
...
...
@@ -13,8 +13,9 @@ namespace server {
ServerError
PrometheusMetrics
::
Init
()
{
ConfigNode
&
configNode
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_METRIC
);
startup_
=
configNode
.
GetValue
(
CONFIG_METRIC_IS_STARTUP
)
==
"true"
?
true
:
false
;
try
{
ConfigNode
&
configNode
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_METRIC
);
startup_
=
configNode
.
GetValue
(
CONFIG_METRIC_IS_STARTUP
)
==
"true"
?
true
:
false
;
// Following should be read from config file.
const
std
::
string
bind_address
=
configNode
.
GetChild
(
CONFIG_PROMETHEUS
).
GetValue
(
CONFIG_METRIC_PROMETHEUS_PORT
);
const
std
::
string
uri
=
std
::
string
(
"/metrics"
);
...
...
@@ -25,6 +26,10 @@ PrometheusMetrics::Init() {
// Exposer Registry
exposer_ptr_
->
RegisterCollectable
(
registry_
);
}
catch
(
std
::
exception
&
ex
)
{
SERVER_LOG_ERROR
<<
"Failed to connect prometheus server: "
<<
std
::
string
(
ex
.
what
());
return
SERVER_UNEXPECTED_ERROR
;
}
return
SERVER_SUCCESS
;
}
...
...
cpp/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @
964d3f37
...
...
@@ -75,6 +75,18 @@ namespace {
return
str
;
}
std
::
string
CurrentTmDate
()
{
time_t
tt
;
time
(
&
tt
);
tt
=
tt
+
8
*
3600
;
tm
*
t
=
gmtime
(
&
tt
);
std
::
string
str
=
std
::
to_string
(
t
->
tm_year
+
1900
)
+
"-"
+
std
::
to_string
(
t
->
tm_mon
+
1
)
+
"-"
+
std
::
to_string
(
t
->
tm_mday
);
return
str
;
}
std
::
string
GetTableName
()
{
static
std
::
string
s_id
(
CurrentTime
());
return
s_id
;
...
...
@@ -170,6 +182,10 @@ ClientTest::Test(const std::string& address, const std::string& port) {
BuildVectors
(
SEARCH_TARGET
,
SEARCH_TARGET
+
10
,
record_array
);
std
::
vector
<
Range
>
query_range_array
;
Range
rg
;
rg
.
start_value
=
CurrentTmDate
();
rg
.
end_value
=
CurrentTmDate
();
query_range_array
.
emplace_back
(
rg
);
std
::
vector
<
TopKQueryResult
>
topk_query_result_array
;
Status
stat
=
conn
->
SearchVector
(
TABLE_NAME
,
record_array
,
query_range_array
,
TOP_K
,
topk_query_result_array
);
std
::
cout
<<
"SearchVector function call status: "
<<
stat
.
ToString
()
<<
std
::
endl
;
...
...
cpp/src/sdk/src/client/ClientProxy.cpp
浏览文件 @
964d3f37
...
...
@@ -159,6 +159,8 @@ ClientProxy::SearchVector(const std::string &table_name,
}
try
{
//step 1: convert vectors data
std
::
vector
<
thrift
::
RowRecord
>
thrift_records
;
for
(
auto
&
record
:
query_record_array
)
{
thrift
::
RowRecord
thrift_record
;
...
...
@@ -172,10 +174,21 @@ ClientProxy::SearchVector(const std::string &table_name,
thrift_records
.
emplace_back
(
thrift_record
);
}
//step 2: convert range array
std
::
vector
<
thrift
::
Range
>
thrift_ranges
;
for
(
auto
&
range
:
query_range_array
)
{
thrift
::
Range
thrift_range
;
thrift_range
.
__set_start_value
(
range
.
start_value
);
thrift_range
.
__set_end_value
(
range
.
end_value
);
thrift_ranges
.
emplace_back
(
thrift_range
);
}
//step 3: search vectors
std
::
vector
<
thrift
::
TopKQueryResult
>
result_array
;
ClientPtr
()
->
interface
()
->
SearchVector
(
result_array
,
table_name
,
thrift_records
,
thrift_ranges
,
topk
);
//step 4: convert result array
for
(
auto
&
thrift_topk_result
:
result_array
)
{
TopKQueryResult
result
;
...
...
cpp/src/server/MegasearchTask.cpp
浏览文件 @
964d3f37
...
...
@@ -75,6 +75,74 @@ namespace {
return
map_type
[
type
];
}
ServerError
ConvertRowRecordToFloatArray
(
const
std
::
vector
<
thrift
::
RowRecord
>&
record_array
,
uint64_t
dimension
,
std
::
vector
<
float
>&
float_array
)
{
ServerError
error_code
;
uint64_t
vec_count
=
record_array
.
size
();
float_array
.
resize
(
vec_count
*
dimension
);
//allocate enough memory
for
(
uint64_t
i
=
0
;
i
<
vec_count
;
i
++
)
{
const
auto
&
record
=
record_array
[
i
];
if
(
record
.
vector_data
.
empty
())
{
error_code
=
SERVER_INVALID_ARGUMENT
;
SERVER_LOG_ERROR
<<
"No vector provided in record"
;
return
error_code
;
}
uint64_t
vec_dim
=
record
.
vector_data
.
size
()
/
sizeof
(
double
);
//how many double value?
if
(
vec_dim
!=
dimension
)
{
SERVER_LOG_ERROR
<<
"Invalid vector dimension: "
<<
vec_dim
<<
" vs. group dimension:"
<<
dimension
;
error_code
=
SERVER_INVALID_VECTOR_DIMENSION
;
return
error_code
;
}
//convert double array to float array(thrift has no float type)
const
double
*
d_p
=
reinterpret_cast
<
const
double
*>
(
record
.
vector_data
.
data
());
for
(
uint64_t
d
=
0
;
d
<
vec_dim
;
d
++
)
{
float_array
[
i
*
vec_dim
+
d
]
=
(
float
)(
d_p
[
d
]);
}
}
return
SERVER_SUCCESS
;
}
static
constexpr
long
DAY_SECONDS
=
86400
;
ServerError
ConvertTimeRangeToDBDates
(
const
std
::
vector
<
megasearch
::
thrift
::
Range
>
&
range_array
,
std
::
vector
<
DB_DATE
>&
dates
)
{
dates
.
clear
();
ServerError
error_code
;
for
(
auto
&
range
:
range_array
)
{
time_t
tt_start
,
tt_end
;
tm
tm_start
,
tm_end
;
if
(
!
CommonUtil
::
TimeStrToTime
(
range
.
start_value
,
tt_start
,
tm_start
)){
error_code
=
SERVER_INVALID_TIME_RANGE
;
SERVER_LOG_ERROR
<<
"Invalid time range: "
<<
range
.
start_value
;
return
error_code
;
}
if
(
!
CommonUtil
::
TimeStrToTime
(
range
.
end_value
,
tt_end
,
tm_end
)){
error_code
=
SERVER_INVALID_TIME_RANGE
;
SERVER_LOG_ERROR
<<
"Invalid time range: "
<<
range
.
end_value
;
return
error_code
;
}
long
days
=
(
tt_end
>
tt_start
)
?
(
tt_end
-
tt_start
)
/
DAY_SECONDS
:
(
tt_start
-
tt_end
)
/
DAY_SECONDS
;
for
(
long
i
=
0
;
i
<=
days
;
i
++
)
{
time_t
tt_day
=
tt_start
+
DAY_SECONDS
*
i
;
tm
tm_day
;
CommonUtil
::
ConvertTime
(
tt_day
,
tm_day
);
long
date
=
tm_day
.
tm_year
*
10000
+
tm_day
.
tm_mon
*
100
+
tm_day
.
tm_mday
;
//according to db logic
dates
.
push_back
(
date
);
}
}
return
SERVER_SUCCESS
;
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
...
...
@@ -237,37 +305,17 @@ ServerError AddVectorTask::OnExecute() {
rc
.
Record
(
"check validation"
);
//step 2: prepare float data
uint64_t
vec_count
=
(
uint64_t
)
record_array_
.
size
();
uint64_t
group_dim
=
table_info
.
dimension_
;
std
::
vector
<
float
>
vec_f
;
vec_f
.
resize
(
vec_count
*
group_dim
);
//allocate enough memory
for
(
uint64_t
i
=
0
;
i
<
vec_count
;
i
++
)
{
const
auto
&
record
=
record_array_
[
i
];
if
(
record
.
vector_data
.
empty
())
{
error_code_
=
SERVER_INVALID_ARGUMENT
;
error_msg_
=
"No vector provided in record"
;
SERVER_LOG_ERROR
<<
error_msg_
;
return
error_code_
;
}
uint64_t
vec_dim
=
record
.
vector_data
.
size
()
/
sizeof
(
double
);
//how many double value?
if
(
vec_dim
!=
group_dim
)
{
SERVER_LOG_ERROR
<<
"Invalid vector dimension: "
<<
vec_dim
<<
" vs. group dimension:"
<<
group_dim
;
error_code_
=
SERVER_INVALID_VECTOR_DIMENSION
;
error_msg_
=
"Engine failed: "
+
stat
.
ToString
();
error_code_
=
ConvertRowRecordToFloatArray
(
record_array_
,
table_info
.
dimension_
,
vec_f
);
if
(
error_code_
!=
SERVER_SUCCESS
)
{
error_msg_
=
"Invalid row record data"
;
return
error_code_
;
}
//convert double array to float array(thrift has no float type)
const
double
*
d_p
=
reinterpret_cast
<
const
double
*>
(
record
.
vector_data
.
data
());
for
(
uint64_t
d
=
0
;
d
<
vec_dim
;
d
++
)
{
vec_f
[
i
*
vec_dim
+
d
]
=
(
float
)(
d_p
[
d
]);
}
}
rc
.
Record
(
"prepare vectors data"
);
//step 3: insert vectors
uint64_t
vec_count
=
(
uint64_t
)
record_array_
.
size
();
stat
=
DB
()
->
InsertVectors
(
table_name_
,
vec_count
,
vec_f
.
data
(),
record_ids_
);
rc
.
Record
(
"add vectors to engine"
);
if
(
!
stat
.
ok
())
{
...
...
@@ -342,44 +390,29 @@ ServerError SearchVectorTask::OnExecute() {
return
error_code_
;
}
//step 3: check date range, and convert to db dates
std
::
vector
<
DB_DATE
>
dates
;
error_code_
=
ConvertTimeRangeToDBDates
(
range_array_
,
dates
);
if
(
error_code_
!=
SERVER_SUCCESS
)
{
error_msg_
=
"Invalid query range"
;
return
error_code_
;
}
rc
.
Record
(
"check validation"
);
//step 3: prepare float data
std
::
vector
<
float
>
vec_f
;
uint64_t
record_count
=
(
uint64_t
)
record_array_
.
size
();
vec_f
.
resize
(
record_count
*
table_info
.
dimension_
);
for
(
uint64_t
i
=
0
;
i
<
record_array_
.
size
();
i
++
)
{
const
auto
&
record
=
record_array_
[
i
];
if
(
record
.
vector_data
.
empty
())
{
error_code_
=
SERVER_INVALID_ARGUMENT
;
error_msg_
=
"Query record has no vector"
;
SERVER_LOG_ERROR
<<
error_msg_
;
error_code_
=
ConvertRowRecordToFloatArray
(
record_array_
,
table_info
.
dimension_
,
vec_f
);
if
(
error_code_
!=
SERVER_SUCCESS
)
{
error_msg_
=
"Invalid row record data"
;
return
error_code_
;
}
uint64_t
vec_dim
=
record
.
vector_data
.
size
()
/
sizeof
(
double
);
//how many double value?
if
(
vec_dim
!=
table_info
.
dimension_
)
{
SERVER_LOG_ERROR
<<
"Invalid vector dimension: "
<<
vec_dim
<<
" vs. group dimension:"
<<
table_info
.
dimension_
;
error_code_
=
SERVER_INVALID_VECTOR_DIMENSION
;
error_msg_
=
"Engine failed: "
+
stat
.
ToString
();
return
error_code_
;
}
//convert double array to float array(thrift has no float type)
const
double
*
d_p
=
reinterpret_cast
<
const
double
*>
(
record
.
vector_data
.
data
());
for
(
uint64_t
d
=
0
;
d
<
vec_dim
;
d
++
)
{
vec_f
[
i
*
vec_dim
+
d
]
=
(
float
)(
d_p
[
d
]);
}
}
rc
.
Record
(
"prepare vector data"
);
//step 4: search vectors
std
::
vector
<
DB_DATE
>
dates
;
engine
::
QueryResults
results
;
uint64_t
record_count
=
(
uint64_t
)
record_array_
.
size
();
stat
=
DB
()
->
Query
(
table_name_
,
(
size_t
)
top_k_
,
record_count
,
vec_f
.
data
(),
dates
,
results
);
rc
.
Record
(
"search vectors from engine"
);
if
(
!
stat
.
ok
())
{
...
...
cpp/src/utils/CommonUtil.cpp
浏览文件 @
964d3f37
...
...
@@ -14,6 +14,7 @@
#include <dirent.h>
#include <string.h>
#include <iostream>
#include <time.h>
#include "boost/filesystem.hpp"
...
...
@@ -150,15 +151,39 @@ std::string CommonUtil::GetExePath() {
return
exe_path
;
}
void
CommonUtil
::
ConvertTime
(
int
year
,
int
month
,
int
day
,
int
hour
,
int
minute
,
int
second
,
time_t
&
t_t
)
{
tm
t_m
;
t_m
.
tm_year
=
year
;
t_m
.
tm_mon
=
month
;
t_m
.
tm_mday
=
day
;
t_m
.
tm_hour
=
hour
;
t_m
.
tm_min
=
minute
;
t_m
.
tm_sec
=
second
;
t_t
=
mktime
(
&
t_m
);
bool
CommonUtil
::
TimeStrToTime
(
const
std
::
string
&
time_str
,
time_t
&
time_integer
,
tm
&
time_struct
,
const
std
::
string
&
format
)
{
time_integer
=
0
;
memset
(
&
time_struct
,
0
,
sizeof
(
tm
));
int
ret
=
sscanf
(
time_str
.
c_str
(),
format
.
c_str
(),
&
(
time_struct
.
tm_year
),
&
(
time_struct
.
tm_mon
),
&
(
time_struct
.
tm_mday
),
&
(
time_struct
.
tm_hour
),
&
(
time_struct
.
tm_min
),
&
(
time_struct
.
tm_sec
));
if
(
ret
<=
0
)
{
return
false
;
}
time_struct
.
tm_year
-=
1900
;
time_struct
.
tm_mon
--
;
time_integer
=
mktime
(
&
time_struct
);
return
true
;
}
void
CommonUtil
::
ConvertTime
(
time_t
time_integer
,
tm
&
time_struct
)
{
tm
*
t_m
=
localtime
(
&
time_integer
);
memcpy
(
&
time_struct
,
t_m
,
sizeof
(
tm
));
}
void
ConvertTime
(
tm
time_struct
,
time_t
&
time_integer
)
{
time_integer
=
mktime
(
&
time_struct
);
}
}
...
...
cpp/src/utils/CommonUtil.h
浏览文件 @
964d3f37
...
...
@@ -26,7 +26,13 @@ class CommonUtil {
static
std
::
string
GetExePath
();
static
void
ConvertTime
(
int
year
,
int
month
,
int
day
,
int
hour
,
int
minute
,
int
second
,
time_t
&
t_t
);
static
bool
TimeStrToTime
(
const
std
::
string
&
time_str
,
time_t
&
time_integer
,
tm
&
time_struct
,
const
std
::
string
&
format
=
"%d-%d-%d %d:%d:%d"
);
static
void
ConvertTime
(
time_t
time_integer
,
tm
&
time_struct
);
static
void
ConvertTime
(
tm
time_struct
,
time_t
&
time_integer
);
};
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录