Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
fbae790c
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
fbae790c
编写于
6月 30, 2019
作者:
J
jinhai
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-133 Change score to distance
Former-commit-id: 6cb53e9fe988a17c293aa9faee663d5d48c98fc9
上级
c3d527f9
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
119 addition
and
128 deletion
+119
-128
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+8
-67
cpp/src/db/DBImpl.h
cpp/src/db/DBImpl.h
+69
-32
cpp/src/db/Status.h
cpp/src/db/Status.h
+22
-9
cpp/src/sdk/examples/simple/src/ClientTest.cpp
cpp/src/sdk/examples/simple/src/ClientTest.cpp
+1
-1
cpp/src/sdk/include/MilvusApi.h
cpp/src/sdk/include/MilvusApi.h
+1
-1
cpp/src/sdk/src/client/ClientProxy.cpp
cpp/src/sdk/src/client/ClientProxy.cpp
+1
-1
cpp/src/thrift/gen-cpp/milvus_types.cpp
cpp/src/thrift/gen-cpp/milvus_types.cpp
+10
-10
cpp/src/thrift/gen-cpp/milvus_types.h
cpp/src/thrift/gen-cpp/milvus_types.h
+6
-6
cpp/src/thrift/milvus.thrift
cpp/src/thrift/milvus.thrift
+1
-1
未找到文件。
cpp/src/db/DBImpl.cpp
浏览文件 @
fbae790c
...
...
@@ -9,7 +9,7 @@
#include "EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/TaskScheduler.h"
#include "scheduler/context/SearchContext.h"
#include "scheduler/context/DeleteContext.h"
#include "utils/TimeRecorder.h"
...
...
@@ -27,9 +27,9 @@ namespace engine {
namespace
{
static
constexpr
uint64_t
METRIC_ACTION_INTERVAL
=
1
;
static
constexpr
uint64_t
COMPACT_ACTION_INTERVAL
=
1
;
static
constexpr
uint64_t
INDEX_ACTION_INTERVAL
=
1
;
constexpr
uint64_t
METRIC_ACTION_INTERVAL
=
1
;
constexpr
uint64_t
COMPACT_ACTION_INTERVAL
=
1
;
constexpr
uint64_t
INDEX_ACTION_INTERVAL
=
1
;
void
CollectInsertMetrics
(
double
total_time
,
size_t
n
,
bool
succeed
)
{
double
avg_time
=
total_time
/
n
;
...
...
@@ -76,56 +76,6 @@ void CollectFileMetrics(int file_type, size_t file_size, double total_time) {
}
}
}
void
CalcScore
(
uint64_t
vector_count
,
const
float
*
vectors_data
,
uint64_t
dimension
,
const
SearchContext
::
ResultSet
&
result_src
,
SearchContext
::
ResultSet
&
result_target
)
{
result_target
.
clear
();
if
(
result_src
.
empty
()){
return
;
}
server
::
TimeRecorder
rc
(
"Calculate Score"
);
int
vec_index
=
0
;
for
(
auto
&
result
:
result_src
)
{
const
float
*
vec_data
=
vectors_data
+
vec_index
*
dimension
;
double
vec_len
=
0
;
for
(
uint64_t
i
=
0
;
i
<
dimension
;
i
++
)
{
vec_len
+=
vec_data
[
i
]
*
vec_data
[
i
];
}
vec_index
++
;
double
max_score
=
0.0
;
for
(
auto
&
pair
:
result
)
{
if
(
max_score
<
pair
.
second
)
{
max_score
=
pair
.
second
;
}
}
//makesure socre is less than 100
if
(
max_score
>
vec_len
)
{
vec_len
=
max_score
;
}
//avoid divided by zero
static
constexpr
double
TOLERANCE
=
std
::
numeric_limits
<
float
>::
epsilon
();
if
(
vec_len
<
TOLERANCE
)
{
vec_len
=
TOLERANCE
;
}
SearchContext
::
Id2ScoreMap
score_array
;
double
vec_len_inverse
=
1.0
/
vec_len
;
for
(
auto
&
pair
:
result
)
{
score_array
.
push_back
(
std
::
make_pair
(
pair
.
first
,
(
1
-
pair
.
second
*
vec_len_inverse
)
*
100.0
));
}
result_target
.
emplace_back
(
score_array
);
}
rc
.
Elapse
(
"totally cost"
);
}
}
...
...
@@ -232,7 +182,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
meta
::
TableFileSchema
table_file
;
table_file
.
table_id_
=
table_id
;
std
::
string
::
size_type
sz
;
ids
.
push_back
(
std
::
stol
(
id
,
&
sz
));
ids
.
push_back
(
std
::
sto
u
l
(
id
,
&
sz
));
}
meta
::
TableFilesSchema
files_array
;
...
...
@@ -380,10 +330,6 @@ Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq,
return
Status
::
NotFound
(
"Group "
+
table_id
+
", search result not found!"
);
}
QueryResults
temp_results
;
CalcScore
(
nq
,
vectors
,
dim
,
results
,
temp_results
);
results
.
swap
(
temp_results
);
return
Status
::
OK
();
}
...
...
@@ -405,13 +351,8 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
context
->
WaitResult
();
//step 3: construct results, calculate score between 0 ~ 100
auto
&
context_result
=
context
->
GetResult
();
meta
::
TableSchema
table_schema
;
table_schema
.
table_id_
=
table_id
;
meta_ptr_
->
DescribeTable
(
table_schema
);
CalcScore
(
context
->
nq
(),
context
->
vectors
(),
table_schema
.
dimension_
,
context_result
,
results
);
//step 3: construct results
results
=
context
->
GetResult
();
return
Status
::
OK
();
}
...
...
@@ -575,7 +516,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
void
DBImpl
::
BackgroundCompaction
(
std
::
set
<
std
::
string
>
table_ids
)
{
Status
status
;
for
(
auto
table_id
:
table_ids
)
{
for
(
auto
&
table_id
:
table_ids
)
{
status
=
BackgroundMergeFiles
(
table_id
);
if
(
!
status
.
ok
())
{
bg_error_
=
status
;
...
...
cpp/src/db/DBImpl.h
浏览文件 @
fbae790c
...
...
@@ -17,6 +17,8 @@
#include <thread>
#include <list>
#include <set>
#include "scheduler/context/SearchContext.h"
namespace
zilliz
{
namespace
milvus
{
...
...
@@ -25,49 +27,80 @@ namespace engine {
class
Env
;
namespace
meta
{
class
Meta
;
class
Meta
;
}
class
DBImpl
:
public
DB
{
public:
public:
using
MetaPtr
=
meta
::
Meta
::
Ptr
;
using
MemManagerPtr
=
typename
MemManager
::
Ptr
;
DBImpl
(
const
Options
&
options
);
explicit
DBImpl
(
const
Options
&
options
);
Status
CreateTable
(
meta
::
TableSchema
&
table_schema
)
override
;
Status
DeleteTable
(
const
std
::
string
&
table_id
,
const
meta
::
DatesT
&
dates
)
override
;
Status
DescribeTable
(
meta
::
TableSchema
&
table_schema
)
override
;
virtual
Status
CreateTable
(
meta
::
TableSchema
&
table_schema
)
override
;
virtual
Status
DeleteTable
(
const
std
::
string
&
table_id
,
const
meta
::
DatesT
&
dates
)
override
;
virtual
Status
DescribeTable
(
meta
::
TableSchema
&
table_schema
)
override
;
virtual
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
virtual
Status
AllTables
(
std
::
vector
<
meta
::
TableSchema
>&
table_schema_array
)
override
;
virtual
Status
GetTableRowCount
(
const
std
::
string
&
table_id
,
uint64_t
&
row_count
)
override
;
Status
HasTable
(
const
std
::
string
&
table_id
,
bool
&
has_or_not
)
override
;
virtual
Status
InsertVectors
(
const
std
::
string
&
table_id
,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_ids
)
override
;
Status
AllTables
(
std
::
vector
<
meta
::
TableSchema
>
&
table_schema_array
)
override
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
QueryResults
&
results
)
override
;
Status
GetTableRowCount
(
const
std
::
string
&
table_id
,
uint64_t
&
row_count
)
override
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
result
s
)
override
;
Status
InsertVectors
(
const
std
::
string
&
table_id
,
uint64_t
n
,
const
float
*
vectors
,
IDNumbers
&
vector_id
s
)
override
;
virtual
Status
Query
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
std
::
string
>&
file_ids
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
override
;
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
QueryResults
&
results
)
override
;
virtual
Status
DropAll
()
override
;
Status
Query
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
override
;
virtual
Status
Size
(
uint64_t
&
result
)
override
;
Status
Query
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
std
::
string
>
&
file_ids
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
)
override
;
virtual
~
DBImpl
()
;
Status
DropAll
()
override
;
private:
Status
QuerySync
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
);
Status
Size
(
uint64_t
&
result
)
override
;
Status
QueryAsync
(
const
std
::
string
&
table_id
,
const
meta
::
TableFilesSchema
&
files
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
);
~
DBImpl
()
override
;
private:
Status
QuerySync
(
const
std
::
string
&
table_id
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
);
Status
QueryAsync
(
const
std
::
string
&
table_id
,
const
meta
::
TableFilesSchema
&
files
,
uint64_t
k
,
uint64_t
nq
,
const
float
*
vectors
,
const
meta
::
DatesT
&
dates
,
QueryResults
&
results
);
void
StartTimerTasks
();
...
...
@@ -76,15 +109,19 @@ private:
void
StartMetricTask
();
void
StartCompactionTask
();
Status
MergeFiles
(
const
std
::
string
&
table_id
,
const
meta
::
DateT
&
date
,
const
meta
::
TableFilesSchema
&
files
);
Status
BackgroundMergeFiles
(
const
std
::
string
&
table_id
);
Status
MergeFiles
(
const
std
::
string
&
table_id
,
const
meta
::
DateT
&
date
,
const
meta
::
TableFilesSchema
&
files
);
Status
BackgroundMergeFiles
(
const
std
::
string
&
table_id
);
void
BackgroundCompaction
(
std
::
set
<
std
::
string
>
table_ids
);
void
StartBuildIndexTask
();
void
BackgroundBuildIndex
();
Status
BuildIndex
(
const
meta
::
TableFileSchema
&
);
Status
BuildIndex
(
const
meta
::
TableFileSchema
&
);
private:
const
Options
options_
;
...
...
cpp/src/db/Status.h
浏览文件 @
fbae790c
...
...
@@ -15,33 +15,46 @@ namespace engine {
class
Status
{
public:
Status
()
noexcept
:
state_
(
nullptr
)
{}
~
Status
()
{
delete
[]
state_
;
}
Status
(
const
Status
&
rhs
);
Status
&
operator
=
(
const
Status
&
rhs
);
Status
&
operator
=
(
const
Status
&
rhs
);
Status
(
Status
&&
rhs
)
noexcept
:
state_
(
rhs
.
state_
)
{
rhs
.
state_
=
nullptr
;
}
Status
&
operator
=
(
Status
&&
rhs_
)
noexcept
;
static
Status
OK
()
{
return
Status
();
}
static
Status
NotFound
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
Status
&
operator
=
(
Status
&&
rhs_
)
noexcept
;
static
Status
OK
()
{
return
Status
();
}
static
Status
NotFound
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kNotFound
,
msg
,
msg2
);
}
static
Status
Error
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
static
Status
Error
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kError
,
msg
,
msg2
);
}
static
Status
InvalidDBPath
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
static
Status
InvalidDBPath
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kInvalidDBPath
,
msg
,
msg2
);
}
static
Status
GroupError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
static
Status
GroupError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kGroupError
,
msg
,
msg2
);
}
static
Status
DBTransactionError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
static
Status
DBTransactionError
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kDBTransactionError
,
msg
,
msg2
);
}
static
Status
AlreadyExist
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
static
Status
AlreadyExist
(
const
std
::
string
&
msg
,
const
std
::
string
&
msg2
=
""
)
{
return
Status
(
kAlreadyExist
,
msg
,
msg2
);
}
...
...
cpp/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @
fbae790c
...
...
@@ -56,7 +56,7 @@ namespace {
<<
std
::
to_string
(
result
.
query_result_arrays
.
size
())
<<
" search result:"
<<
std
::
endl
;
for
(
auto
&
item
:
result
.
query_result_arrays
)
{
std
::
cout
<<
"
\t
"
<<
std
::
to_string
(
item
.
id
)
<<
"
\t
score:"
<<
std
::
to_string
(
item
.
scor
e
);
std
::
cout
<<
"
\t
"
<<
std
::
to_string
(
item
.
id
)
<<
"
\t
distance:"
<<
std
::
to_string
(
item
.
distanc
e
);
std
::
cout
<<
std
::
endl
;
}
}
...
...
cpp/src/sdk/include/MilvusApi.h
浏览文件 @
fbae790c
...
...
@@ -59,7 +59,7 @@ struct RowRecord {
*/
struct
QueryResult
{
int64_t
id
;
///< Output result
double
score
;
///< Vector similarity score: 0 ~ 100
double
distance
;
///< Vector similarity distance
};
/**
...
...
cpp/src/sdk/src/client/ClientProxy.cpp
浏览文件 @
fbae790c
...
...
@@ -204,7 +204,7 @@ ClientProxy::SearchVector(const std::string &table_name,
for
(
auto
&
thrift_query_result
:
thrift_topk_result
.
query_result_arrays
)
{
QueryResult
query_result
;
query_result
.
id
=
thrift_query_result
.
id
;
query_result
.
score
=
thrift_query_result
.
scor
e
;
query_result
.
distance
=
thrift_query_result
.
distanc
e
;
result
.
query_result_arrays
.
emplace_back
(
query_result
);
}
...
...
cpp/src/thrift/gen-cpp/milvus_types.cpp
浏览文件 @
fbae790c
...
...
@@ -536,8 +536,8 @@ void QueryResult::__set_id(const int64_t val) {
this
->
id
=
val
;
}
void
QueryResult
::
__set_
scor
e
(
const
double
val
)
{
this
->
scor
e
=
val
;
void
QueryResult
::
__set_
distanc
e
(
const
double
val
)
{
this
->
distanc
e
=
val
;
}
std
::
ostream
&
operator
<<
(
std
::
ostream
&
out
,
const
QueryResult
&
obj
)
{
...
...
@@ -577,8 +577,8 @@ uint32_t QueryResult::read(::apache::thrift::protocol::TProtocol* iprot) {
break
;
case
2
:
if
(
ftype
==
::
apache
::
thrift
::
protocol
::
T_DOUBLE
)
{
xfer
+=
iprot
->
readDouble
(
this
->
scor
e
);
this
->
__isset
.
scor
e
=
true
;
xfer
+=
iprot
->
readDouble
(
this
->
distanc
e
);
this
->
__isset
.
distanc
e
=
true
;
}
else
{
xfer
+=
iprot
->
skip
(
ftype
);
}
...
...
@@ -604,8 +604,8 @@ uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const
xfer
+=
oprot
->
writeI64
(
this
->
id
);
xfer
+=
oprot
->
writeFieldEnd
();
xfer
+=
oprot
->
writeFieldBegin
(
"
scor
e"
,
::
apache
::
thrift
::
protocol
::
T_DOUBLE
,
2
);
xfer
+=
oprot
->
writeDouble
(
this
->
scor
e
);
xfer
+=
oprot
->
writeFieldBegin
(
"
distanc
e"
,
::
apache
::
thrift
::
protocol
::
T_DOUBLE
,
2
);
xfer
+=
oprot
->
writeDouble
(
this
->
distanc
e
);
xfer
+=
oprot
->
writeFieldEnd
();
xfer
+=
oprot
->
writeFieldStop
();
...
...
@@ -616,18 +616,18 @@ uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const
void
swap
(
QueryResult
&
a
,
QueryResult
&
b
)
{
using
::
std
::
swap
;
swap
(
a
.
id
,
b
.
id
);
swap
(
a
.
score
,
b
.
scor
e
);
swap
(
a
.
distance
,
b
.
distanc
e
);
swap
(
a
.
__isset
,
b
.
__isset
);
}
QueryResult
::
QueryResult
(
const
QueryResult
&
other9
)
{
id
=
other9
.
id
;
score
=
other9
.
scor
e
;
distance
=
other9
.
distanc
e
;
__isset
=
other9
.
__isset
;
}
QueryResult
&
QueryResult
::
operator
=
(
const
QueryResult
&
other10
)
{
id
=
other10
.
id
;
score
=
other10
.
scor
e
;
distance
=
other10
.
distanc
e
;
__isset
=
other10
.
__isset
;
return
*
this
;
}
...
...
@@ -635,7 +635,7 @@ void QueryResult::printTo(std::ostream& out) const {
using
::
apache
::
thrift
::
to_string
;
out
<<
"QueryResult("
;
out
<<
"id="
<<
to_string
(
id
);
out
<<
", "
<<
"
score="
<<
to_string
(
scor
e
);
out
<<
", "
<<
"
distance="
<<
to_string
(
distanc
e
);
out
<<
")"
;
}
...
...
cpp/src/thrift/gen-cpp/milvus_types.h
浏览文件 @
fbae790c
...
...
@@ -242,9 +242,9 @@ void swap(RowRecord &a, RowRecord &b);
std
::
ostream
&
operator
<<
(
std
::
ostream
&
out
,
const
RowRecord
&
obj
);
typedef
struct
_QueryResult__isset
{
_QueryResult__isset
()
:
id
(
false
),
scor
e
(
false
)
{}
_QueryResult__isset
()
:
id
(
false
),
distanc
e
(
false
)
{}
bool
id
:
1
;
bool
scor
e
:
1
;
bool
distanc
e
:
1
;
}
_QueryResult__isset
;
class
QueryResult
:
public
virtual
::
apache
::
thrift
::
TBase
{
...
...
@@ -252,24 +252,24 @@ class QueryResult : public virtual ::apache::thrift::TBase {
QueryResult
(
const
QueryResult
&
);
QueryResult
&
operator
=
(
const
QueryResult
&
);
QueryResult
()
:
id
(
0
),
scor
e
(
0
)
{
QueryResult
()
:
id
(
0
),
distanc
e
(
0
)
{
}
virtual
~
QueryResult
()
throw
();
int64_t
id
;
double
scor
e
;
double
distanc
e
;
_QueryResult__isset
__isset
;
void
__set_id
(
const
int64_t
val
);
void
__set_
scor
e
(
const
double
val
);
void
__set_
distanc
e
(
const
double
val
);
bool
operator
==
(
const
QueryResult
&
rhs
)
const
{
if
(
!
(
id
==
rhs
.
id
))
return
false
;
if
(
!
(
score
==
rhs
.
scor
e
))
if
(
!
(
distance
==
rhs
.
distanc
e
))
return
false
;
return
true
;
}
...
...
cpp/src/thrift/milvus.thrift
浏览文件 @
fbae790c
...
...
@@ -59,7 +59,7 @@ struct RowRecord {
*/
struct QueryResult {
1: i64 id; ///< Output result
2: double
score; ///< Vector similarity score: 0 ~ 100
2: double
distance; ///< Vector similarity distance
}
/**
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录