Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
766c8f99
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
766c8f99
编写于
10月 14, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-624 Search vectors failed if time ranges long enough
Former-commit-id: ef8933fcce46e08184581e9dfb8f6dd46e65f86e
上级
e037b64d
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
79 addition
and
28 deletion
+79
-28
core/CHANGELOG.md
core/CHANGELOG.md
+3
-2
core/src/db/meta/Meta.h
core/src/db/meta/Meta.h
+1
-1
core/src/db/meta/MySQLMetaImpl.cpp
core/src/db/meta/MySQLMetaImpl.cpp
+3
-3
core/src/db/meta/MySQLMetaImpl.h
core/src/db/meta/MySQLMetaImpl.h
+1
-1
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+70
-20
core/src/db/meta/SqliteMetaImpl.h
core/src/db/meta/SqliteMetaImpl.h
+1
-1
未找到文件。
core/CHANGELOG.md
浏览文件 @
766c8f99
...
@@ -9,14 +9,15 @@ Please mark all change in change log and use the ticket from JIRA.
...
@@ -9,14 +9,15 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-572 - Milvus crash when get SIGINT
-
MS-572 - Milvus crash when get SIGINT
-
MS-577 - Unittest Query randomly hung
-
MS-577 - Unittest Query randomly hung
-
MS-587 - Count get wrong result after adding vectors and index built immediately
-
MS-587 - Count get wrong result after adding vectors and index built immediately
-
MS-599 -
s
earch wrong result when table created with metric_type: IP
-
MS-599 -
S
earch wrong result when table created with metric_type: IP
-
MS-601 - Docker logs error caused by get CPUTemperature error
-
MS-601 - Docker logs error caused by get CPUTemperature error
-
MS-622 - Delete vectors should be failed if date range is invalid
-
MS-622 - Delete vectors should be failed if date range is invalid
-
MS-620 - Get table row counts display wrong error code
-
MS-620 - Get table row counts display wrong error code
-
MS-637 -
o
ut of memory when load too many tasks
-
MS-637 -
O
ut of memory when load too many tasks
-
MS-640 - Cache object size calculate incorrect
-
MS-640 - Cache object size calculate incorrect
-
MS-641 - Segment fault(signal 11) in PickToLoad
-
MS-641 - Segment fault(signal 11) in PickToLoad
-
MS-639 - SQ8H index created failed and server hang
-
MS-639 - SQ8H index created failed and server hang
-
MS-624 - Search vectors failed if time ranges long enough
## Improvement
## Improvement
-
MS-552 - Add and change the easylogging library
-
MS-552 - Add and change the easylogging library
...
...
core/src/db/meta/Meta.h
浏览文件 @
766c8f99
...
@@ -81,7 +81,7 @@ class Meta {
...
@@ -81,7 +81,7 @@ class Meta {
UpdateTableFiles
(
TableFilesSchema
&
files
)
=
0
;
UpdateTableFiles
(
TableFilesSchema
&
files
)
=
0
;
virtual
Status
virtual
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
partition
,
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
dates
,
DatePartionedTableFilesSchema
&
files
)
=
0
;
DatePartionedTableFilesSchema
&
files
)
=
0
;
virtual
Status
virtual
Status
...
...
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
766c8f99
...
@@ -1088,7 +1088,7 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) {
...
@@ -1088,7 +1088,7 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) {
}
}
Status
Status
MySQLMetaImpl
::
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
partition
,
MySQLMetaImpl
::
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
dates
,
DatePartionedTableFilesSchema
&
files
)
{
DatePartionedTableFilesSchema
&
files
)
{
files
.
clear
();
files
.
clear
();
...
@@ -1108,9 +1108,9 @@ MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector<size
...
@@ -1108,9 +1108,9 @@ MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector<size
<<
META_TABLEFILES
<<
" "
<<
META_TABLEFILES
<<
" "
<<
"WHERE table_id = "
<<
mysqlpp
::
quote
<<
table_id
;
<<
"WHERE table_id = "
<<
mysqlpp
::
quote
<<
table_id
;
if
(
!
partition
.
empty
())
{
if
(
!
dates
.
empty
())
{
std
::
stringstream
partitionListSS
;
std
::
stringstream
partitionListSS
;
for
(
auto
&
date
:
partition
)
{
for
(
auto
&
date
:
dates
)
{
partitionListSS
<<
std
::
to_string
(
date
)
<<
", "
;
partitionListSS
<<
std
::
to_string
(
date
)
<<
", "
;
}
}
std
::
string
partitionListStr
=
partitionListSS
.
str
();
std
::
string
partitionListStr
=
partitionListSS
.
str
();
...
...
core/src/db/meta/MySQLMetaImpl.h
浏览文件 @
766c8f99
...
@@ -89,7 +89,7 @@ class MySQLMetaImpl : public Meta {
...
@@ -89,7 +89,7 @@ class MySQLMetaImpl : public Meta {
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
Status
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
partition
,
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
dates
,
DatePartionedTableFilesSchema
&
files
)
override
;
DatePartionedTableFilesSchema
&
files
)
override
;
Status
Status
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
766c8f99
...
@@ -159,7 +159,7 @@ SqliteMetaImpl::Initialize() {
...
@@ -159,7 +159,7 @@ SqliteMetaImpl::Initialize() {
Status
Status
SqliteMetaImpl
::
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
SqliteMetaImpl
::
DropPartitionsByDates
(
const
std
::
string
&
table_id
,
const
DatesT
&
dates
)
{
const
DatesT
&
dates
)
{
if
(
dates
.
size
()
==
0
)
{
if
(
dates
.
empty
()
)
{
return
Status
::
OK
();
return
Status
::
OK
();
}
}
...
@@ -171,16 +171,35 @@ SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id,
...
@@ -171,16 +171,35 @@ SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id,
}
}
try
{
try
{
//sqlite_orm has a bug, 'in' statement cannot handle too many elements
//so we split one query into multi-queries, this is a work-around!!
std
::
vector
<
DatesT
>
split_dates
;
split_dates
.
push_back
(
DatesT
());
const
size_t
batch_size
=
30
;
for
(
DateT
date
:
dates
)
{
DatesT
&
last_batch
=
*
split_dates
.
rbegin
();
last_batch
.
push_back
(
date
);
if
(
last_batch
.
size
()
>
batch_size
)
{
split_dates
.
push_back
(
DatesT
());
}
}
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std
::
lock_guard
<
std
::
mutex
>
meta_lock
(
meta_mutex_
);
std
::
lock_guard
<
std
::
mutex
>
meta_lock
(
meta_mutex_
);
ConnectorPtr
->
update_all
(
for
(
auto
&
batch_dates
:
split_dates
)
{
set
(
if
(
batch_dates
.
empty
())
{
c
(
&
TableFileSchema
::
file_type_
)
=
(
int
)
TableFileSchema
::
TO_DELETE
,
continue
;
c
(
&
TableFileSchema
::
updated_time_
)
=
utils
::
GetMicroSecTimeStamp
()),
}
where
(
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
and
ConnectorPtr
->
update_all
(
in
(
&
TableFileSchema
::
date_
,
dates
)));
set
(
c
(
&
TableFileSchema
::
file_type_
)
=
(
int
)
TableFileSchema
::
TO_DELETE
,
c
(
&
TableFileSchema
::
updated_time_
)
=
utils
::
GetMicroSecTimeStamp
()),
where
(
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
and
in
(
&
TableFileSchema
::
date_
,
batch_dates
)));
}
ENGINE_LOG_DEBUG
<<
"Successfully drop partitions, table id = "
<<
table_schema
.
table_id_
;
ENGINE_LOG_DEBUG
<<
"Successfully drop partitions, table id = "
<<
table_schema
.
table_id_
;
}
catch
(
std
::
exception
&
e
)
{
}
catch
(
std
::
exception
&
e
)
{
...
@@ -673,7 +692,7 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
...
@@ -673,7 +692,7 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
Status
Status
SqliteMetaImpl
::
FilesToSearch
(
const
std
::
string
&
table_id
,
SqliteMetaImpl
::
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>
&
ids
,
const
std
::
vector
<
size_t
>
&
ids
,
const
DatesT
&
partition
,
const
DatesT
&
dates
,
DatePartionedTableFilesSchema
&
files
)
{
DatePartionedTableFilesSchema
&
files
)
{
files
.
clear
();
files
.
clear
();
server
::
MetricCollector
metric
;
server
::
MetricCollector
metric
;
...
@@ -702,23 +721,54 @@ SqliteMetaImpl::FilesToSearch(const std::string &table_id,
...
@@ -702,23 +721,54 @@ SqliteMetaImpl::FilesToSearch(const std::string &table_id,
auto
status
=
DescribeTable
(
table_schema
);
auto
status
=
DescribeTable
(
table_schema
);
if
(
!
status
.
ok
())
{
return
status
;
}
if
(
!
status
.
ok
())
{
return
status
;
}
//sqlite_orm has a bug, 'in' statement cannot handle too many elements
//so we split one query into multi-queries, this is a work-around!!
std
::
vector
<
DatesT
>
split_dates
;
split_dates
.
push_back
(
DatesT
());
const
size_t
batch_size
=
30
;
for
(
DateT
date
:
dates
)
{
DatesT
&
last_batch
=
*
split_dates
.
rbegin
();
last_batch
.
push_back
(
date
);
if
(
last_batch
.
size
()
>
batch_size
)
{
split_dates
.
push_back
(
DatesT
());
}
}
//perform query
decltype
(
ConnectorPtr
->
select
(
select_columns
))
selected
;
decltype
(
ConnectorPtr
->
select
(
select_columns
))
selected
;
if
(
partition
.
empty
()
&&
ids
.
empty
())
{
if
(
dates
.
empty
()
&&
ids
.
empty
())
{
auto
filter
=
where
(
match_tableid
and
match_type
);
auto
filter
=
where
(
match_tableid
and
match_type
);
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
}
else
if
(
partition
.
empty
()
&&
!
ids
.
empty
())
{
}
else
if
(
dates
.
empty
()
&&
!
ids
.
empty
())
{
auto
match_fileid
=
in
(
&
TableFileSchema
::
id_
,
ids
);
auto
match_fileid
=
in
(
&
TableFileSchema
::
id_
,
ids
);
auto
filter
=
where
(
match_tableid
and
match_fileid
and
match_type
);
auto
filter
=
where
(
match_tableid
and
match_fileid
and
match_type
);
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
}
else
if
(
!
partition
.
empty
()
&&
ids
.
empty
())
{
}
else
if
(
!
dates
.
empty
()
&&
ids
.
empty
())
{
auto
match_date
=
in
(
&
TableFileSchema
::
date_
,
partition
);
for
(
auto
&
batch_dates
:
split_dates
)
{
auto
filter
=
where
(
match_tableid
and
match_date
and
match_type
);
if
(
batch_dates
.
empty
())
{
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
continue
;
}
else
if
(
!
partition
.
empty
()
&&
!
ids
.
empty
())
{
}
auto
match_fileid
=
in
(
&
TableFileSchema
::
id_
,
ids
);
auto
match_date
=
in
(
&
TableFileSchema
::
date_
,
batch_dates
);
auto
match_date
=
in
(
&
TableFileSchema
::
date_
,
partition
);
auto
filter
=
where
(
match_tableid
and
match_date
and
match_type
);
auto
filter
=
where
(
match_tableid
and
match_fileid
and
match_date
and
match_type
);
auto
batch_selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
for
(
auto
&
file
:
selected
)
{
selected
.
push_back
(
file
);
}
}
}
else
if
(
!
dates
.
empty
()
&&
!
ids
.
empty
())
{
for
(
auto
&
batch_dates
:
split_dates
)
{
if
(
batch_dates
.
empty
())
{
continue
;
}
auto
match_fileid
=
in
(
&
TableFileSchema
::
id_
,
ids
);
auto
match_date
=
in
(
&
TableFileSchema
::
date_
,
batch_dates
);
auto
filter
=
where
(
match_tableid
and
match_fileid
and
match_date
and
match_type
);
auto
batch_selected
=
ConnectorPtr
->
select
(
select_columns
,
filter
);
for
(
auto
&
file
:
selected
)
{
selected
.
push_back
(
file
);
}
}
}
}
Status
ret
;
Status
ret
;
...
...
core/src/db/meta/SqliteMetaImpl.h
浏览文件 @
766c8f99
...
@@ -89,7 +89,7 @@ class SqliteMetaImpl : public Meta {
...
@@ -89,7 +89,7 @@ class SqliteMetaImpl : public Meta {
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
UpdateTableFiles
(
TableFilesSchema
&
files
)
override
;
Status
Status
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
partition
,
FilesToSearch
(
const
std
::
string
&
table_id
,
const
std
::
vector
<
size_t
>&
ids
,
const
DatesT
&
dates
,
DatePartionedTableFilesSchema
&
files
)
override
;
DatePartionedTableFilesSchema
&
files
)
override
;
Status
Status
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录