Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
e83c4278
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e83c4278
编写于
8月 21, 2019
作者:
Y
Yu Kun
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add deletebyrange interface and unittest
Former-commit-id: 54a94b0924d2de5089a4f31810e31a765bceb591
上级
c1e82cb7
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
215 addition
and
15 deletion
+215
-15
cpp/src/db/DBImpl.cpp
cpp/src/db/DBImpl.cpp
+11
-6
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
+9
-0
cpp/src/sdk/grpc/ClientProxy.cpp
cpp/src/sdk/grpc/ClientProxy.cpp
+11
-4
cpp/src/sdk/grpc/GrpcClient.cpp
cpp/src/sdk/grpc/GrpcClient.cpp
+18
-0
cpp/src/sdk/interface/ConnectionImpl.cpp
cpp/src/sdk/interface/ConnectionImpl.cpp
+1
-1
cpp/src/server/grpc_impl/GrpcRequestHandler.cpp
cpp/src/server/grpc_impl/GrpcRequestHandler.cpp
+6
-1
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
+67
-0
cpp/unittest/db/db_tests.cpp
cpp/unittest/db/db_tests.cpp
+92
-3
未找到文件。
cpp/src/db/DBImpl.cpp
浏览文件 @
e83c4278
...
...
@@ -104,13 +104,18 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
//dates partly delete files of the table but currently we don't support
ENGINE_LOG_DEBUG
<<
"Prepare to delete table "
<<
table_id
;
mem_mgr_
->
EraseMemVector
(
table_id
);
//not allow insert
meta_ptr_
->
DeleteTable
(
table_id
);
//soft delete table
if
(
dates
.
empty
())
{
mem_mgr_
->
EraseMemVector
(
table_id
);
//not allow insert
meta_ptr_
->
DeleteTable
(
table_id
);
//soft delete table
//scheduler will determine when to delete table files
TaskScheduler
&
scheduler
=
TaskScheduler
::
GetInstance
();
DeleteContextPtr
context
=
std
::
make_shared
<
DeleteContext
>
(
table_id
,
meta_ptr_
);
scheduler
.
Schedule
(
context
);
}
else
{
meta_ptr_
->
DropPartitionsByDates
(
table_id
,
dates
);
}
//scheduler will determine when to delete table files
TaskScheduler
&
scheduler
=
TaskScheduler
::
GetInstance
();
DeleteContextPtr
context
=
std
::
make_shared
<
DeleteContext
>
(
table_id
,
meta_ptr_
);
scheduler
.
Schedule
(
context
);
return
Status
::
OK
();
}
...
...
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
浏览文件 @
e83c4278
...
...
@@ -290,6 +290,15 @@ ClientTest::Test(const std::string& address, const std::string& port) {
DoSearch
(
conn
,
search_record_array
,
"Search after build index finish"
);
}
{
Range
rg
;
rg
.
start_value
=
CurrentTmDate
(
-
2
);
rg
.
end_value
=
CurrentTmDate
(
-
3
);
Status
stat
=
conn
->
DeleteByRange
(
rg
,
TABLE_NAME
);
std
::
cout
<<
"DeleteByRange function call status: "
<<
stat
.
ToString
()
<<
std
::
endl
;
}
{
//delete table
Status
stat
=
conn
->
DropTable
(
TABLE_NAME
);
std
::
cout
<<
"DeleteTable function call status: "
<<
stat
.
ToString
()
<<
std
::
endl
;
...
...
cpp/src/sdk/grpc/ClientProxy.cpp
浏览文件 @
e83c4278
...
...
@@ -330,10 +330,18 @@ ClientProxy::ServerStatus() const {
Status
ClientProxy
::
DeleteByRange
(
milvus
::
Range
&
range
,
const
std
::
string
&
table_name
)
{
try
{
::
milvus
::
grpc
::
DeleteByRangeParam
delete_by_range_param
;
delete_by_range_param
.
set_table_name
(
table_name
);
delete_by_range_param
.
mutable_range
()
->
set_start_value
(
range
.
start_value
);
delete_by_range_param
.
mutable_range
()
->
set_end_value
(
range
.
end_value
);
return
client_ptr_
->
DeleteByRange
(
delete_by_range_param
);
}
catch
(
std
::
exception
&
ex
)
{
return
Status
(
StatusCode
::
UnknownError
,
"fail to delete by range: "
+
std
::
string
(
ex
.
what
()));
}
}
Status
Status
ClientProxy
::
PreloadTable
(
const
std
::
string
&
table_name
)
const
{
try
{
::
milvus
::
grpc
::
TableName
grpc_table_name
;
...
...
@@ -341,13 +349,12 @@ ClientProxy::PreloadTable(const std::string &table_name) const {
Status
status
=
client_ptr_
->
PreloadTable
(
grpc_table_name
);
return
status
;
}
catch
(
std
::
exception
&
ex
)
{
return
Status
(
StatusCode
::
UnknownError
,
"fail to
show
tables: "
+
std
::
string
(
ex
.
what
()));
return
Status
(
StatusCode
::
UnknownError
,
"fail to
preload
tables: "
+
std
::
string
(
ex
.
what
()));
}
}
IndexParam
ClientProxy
::
DescribeIndex
(
const
std
::
string
&
table_name
)
const
{
}
Status
...
...
cpp/src/sdk/grpc/GrpcClient.cpp
浏览文件 @
e83c4278
...
...
@@ -264,6 +264,24 @@ GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) {
return
Status
::
OK
();
}
Status
GrpcClient
::
DeleteByRange
(
grpc
::
DeleteByRangeParam
&
delete_by_range_param
)
{
ClientContext
context
;
::
milvus
::
grpc
::
Status
response
;
::
grpc
::
Status
grpc_status
=
stub_
->
DeleteByRange
(
&
context
,
delete_by_range_param
,
&
response
);
if
(
!
grpc_status
.
ok
())
{
std
::
cerr
<<
"DeleteByRange gRPC failed!"
<<
std
::
endl
;
return
Status
(
StatusCode
::
RPCFailed
,
grpc_status
.
error_message
());
}
if
(
response
.
error_code
()
!=
grpc
::
SUCCESS
)
{
std
::
cerr
<<
response
.
reason
()
<<
std
::
endl
;
return
Status
(
StatusCode
::
ServerFailed
,
response
.
reason
());
}
return
Status
::
OK
();
}
Status
GrpcClient
::
Disconnect
()
{
stub_
.
release
();
...
...
cpp/src/sdk/interface/ConnectionImpl.cpp
浏览文件 @
e83c4278
...
...
@@ -117,7 +117,7 @@ ConnectionImpl::ServerStatus() const {
Status
ConnectionImpl
::
DeleteByRange
(
Range
&
range
,
const
std
::
string
&
table_name
)
{
return
client_proxy_
->
DeleteByRange
(
range
,
table_name
);
}
Status
...
...
cpp/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @
e83c4278
...
...
@@ -168,7 +168,12 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context,
GrpcRequestHandler
::
DeleteByRange
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
DeleteByRangeParam
*
request
,
::
milvus
::
grpc
::
Status
*
response
)
{
BaseTaskPtr
task_ptr
=
DeleteByRangeTask
::
Create
(
*
request
);
::
milvus
::
grpc
::
Status
grpc_status
;
GrpcRequestScheduler
::
ExecTask
(
task_ptr
,
&
grpc_status
);
response
->
set_error_code
(
grpc_status
.
error_code
());
response
->
set_reason
(
grpc_status
.
reason
());
return
::
grpc
::
Status
::
OK
;
}
::
grpc
::
Status
...
...
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @
e83c4278
...
...
@@ -721,6 +721,73 @@ CmdTask::OnExecute() {
return
SERVER_SUCCESS
;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteByRangeTask
::
DeleteByRangeTask
(
const
::
milvus
::
grpc
::
DeleteByRangeParam
&
delete_by_range_param
)
:
GrpcBaseTask
(
DDL_DML_TASK_GROUP
),
delete_by_range_param_
(
delete_by_range_param
){
}
BaseTaskPtr
DeleteByRangeTask
::
Create
(
const
::
milvus
::
grpc
::
DeleteByRangeParam
&
delete_by_range_param
)
{
return
std
::
shared_ptr
<
GrpcBaseTask
>
(
new
DeleteByRangeTask
(
delete_by_range_param
));
}
ServerError
DeleteByRangeTask
::
OnExecute
()
{
try
{
TimeRecorder
rc
(
"DeleteByRangeTask"
);
//step 1: check arguments
std
::
string
table_name
=
delete_by_range_param_
.
table_name
();
ServerError
res
=
ValidationUtil
::
ValidateTableName
(
table_name
);
if
(
res
!=
SERVER_SUCCESS
)
{
return
SetError
(
res
,
"Invalid table name: "
+
table_name
);
}
//step 2: check table existence
engine
::
meta
::
TableSchema
table_info
;
table_info
.
table_id_
=
table_name
;
engine
::
Status
stat
=
DBWrapper
::
DB
()
->
DescribeTable
(
table_info
);
if
(
!
stat
.
ok
())
{
if
(
stat
.
IsNotFound
())
{
return
SetError
(
SERVER_TABLE_NOT_EXIST
,
"Table "
+
table_name
+
" not exists"
);
}
else
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
"Engine failed: "
+
stat
.
ToString
());
}
}
rc
.
ElapseFromBegin
(
"check validation"
);
//step 3: check date range, and convert to db dates
std
::
vector
<
DB_DATE
>
dates
;
ServerError
error_code
=
SERVER_SUCCESS
;
std
::
string
error_msg
;
std
::
vector
<::
milvus
::
grpc
::
Range
>
range_array
;
range_array
.
emplace_back
(
delete_by_range_param_
.
range
());
ConvertTimeRangeToDBDates
(
range_array
,
dates
,
error_code
,
error_msg
);
if
(
error_code
!=
SERVER_SUCCESS
)
{
return
SetError
(
error_code
,
error_msg
);
}
#ifdef MILVUS_ENABLE_PROFILING
std
::
string
fname
=
"/tmp/search_nq_"
+
std
::
to_string
(
this
->
record_array_
.
size
())
+
"_top_"
+
std
::
to_string
(
this
->
top_k_
)
+
"_"
+
GetCurrTimeStr
()
+
".profiling"
;
ProfilerStart
(
fname
.
c_str
());
#endif
engine
::
Status
status
=
DBWrapper
::
DB
()
->
DeleteTable
(
table_name
,
dates
);
if
(
!
status
.
ok
())
{
return
SetError
(
DB_META_TRANSACTION_FAILED
,
"Engine failed: "
+
stat
.
ToString
());
}
}
catch
(
std
::
exception
&
ex
)
{
return
SetError
(
SERVER_UNEXPECTED_ERROR
,
ex
.
what
());
}
return
SERVER_SUCCESS
;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PreloadTableTask
::
PreloadTableTask
(
const
std
::
string
&
table_name
)
:
GrpcBaseTask
(
DDL_DML_TASK_GROUP
),
...
...
cpp/unittest/db/db_tests.cpp
浏览文件 @
e83c4278
...
...
@@ -9,6 +9,7 @@
#include "db/meta/MetaConsts.h"
#include "db/Factories.h"
#include "cache/CpuCacheMgr.h"
#include "utils/CommonUtil.h"
#include <gtest/gtest.h>
#include <easylogging++.h>
...
...
@@ -26,6 +27,8 @@ namespace {
static
constexpr
int64_t
TABLE_DIM
=
256
;
static
constexpr
int64_t
VECTOR_COUNT
=
250000
;
static
constexpr
int64_t
INSERT_LOOP
=
10000
;
static
constexpr
int64_t
SECONDS_EACH_HOUR
=
3600
;
static
constexpr
int64_t
DAY_SECONDS
=
24
*
60
*
60
;
engine
::
meta
::
TableSchema
BuildTableSchema
()
{
engine
::
meta
::
TableSchema
table_info
;
...
...
@@ -45,6 +48,52 @@ namespace {
}
}
std
::
string
CurrentTmDate
(
int64_t
offset_day
=
0
)
{
time_t
tt
;
time
(
&
tt
);
tt
=
tt
+
8
*
SECONDS_EACH_HOUR
;
tt
=
tt
+
24
*
SECONDS_EACH_HOUR
*
offset_day
;
tm
*
t
=
gmtime
(
&
tt
);
std
::
string
str
=
std
::
to_string
(
t
->
tm_year
+
1900
)
+
"-"
+
std
::
to_string
(
t
->
tm_mon
+
1
)
+
"-"
+
std
::
to_string
(
t
->
tm_mday
);
return
str
;
}
void
ConvertTimeRangeToDBDates
(
const
std
::
string
&
start_value
,
const
std
::
string
&
end_value
,
std
::
vector
<
engine
::
meta
::
DateT
>
&
dates
)
{
dates
.
clear
();
time_t
tt_start
,
tt_end
;
tm
tm_start
,
tm_end
;
if
(
!
zilliz
::
milvus
::
server
::
CommonUtil
::
TimeStrToTime
(
start_value
,
tt_start
,
tm_start
))
{
return
;
}
if
(
!
zilliz
::
milvus
::
server
::
CommonUtil
::
TimeStrToTime
(
end_value
,
tt_end
,
tm_end
))
{
return
;
}
long
days
=
(
tt_end
>
tt_start
)
?
(
tt_end
-
tt_start
)
/
DAY_SECONDS
:
(
tt_start
-
tt_end
)
/
DAY_SECONDS
;
if
(
days
==
0
)
{
return
;
}
for
(
long
i
=
0
;
i
<
days
;
i
++
)
{
time_t
tt_day
=
tt_start
+
DAY_SECONDS
*
i
;
tm
tm_day
;
zilliz
::
milvus
::
server
::
CommonUtil
::
ConvertTime
(
tt_day
,
tm_day
);
long
date
=
tm_day
.
tm_year
*
10000
+
tm_day
.
tm_mon
*
100
+
tm_day
.
tm_mday
;
//according to db logic
dates
.
push_back
(
date
);
}
}
}
TEST_F
(
DBTest
,
CONFIG_TEST
)
{
...
...
@@ -307,8 +356,6 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
};
TEST_F
(
DBTest2
,
DELETE_TEST
)
{
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
...
...
@@ -343,4 +390,46 @@ TEST_F(DBTest2, DELETE_TEST) {
db_
->
HasTable
(
TABLE_NAME
,
has_table
);
ASSERT_FALSE
(
has_table
);
};
\ No newline at end of file
};
TEST_F
(
DBTest2
,
DELETE_BY_RANGE_TEST
)
{
auto
options
=
engine
::
OptionsFactory
::
Build
();
options
.
meta
.
path
=
"/tmp/milvus_test"
;
options
.
meta
.
backend_uri
=
"sqlite://:@:/"
;
auto
db_
=
engine
::
DBFactory
::
Build
(
options
);
engine
::
meta
::
TableSchema
table_info
=
BuildTableSchema
();
engine
::
Status
stat
=
db_
->
CreateTable
(
table_info
);
engine
::
meta
::
TableSchema
table_info_get
;
table_info_get
.
table_id_
=
TABLE_NAME
;
stat
=
db_
->
DescribeTable
(
table_info_get
);
ASSERT_STATS
(
stat
);
bool
has_table
=
false
;
db_
->
HasTable
(
TABLE_NAME
,
has_table
);
ASSERT_TRUE
(
has_table
);
engine
::
IDNumbers
vector_ids
;
uint64_t
size
;
db_
->
Size
(
size
);
int64_t
nb
=
INSERT_LOOP
;
std
::
vector
<
float
>
xb
;
BuildVectors
(
nb
,
xb
);
int
loop
=
20
;
for
(
auto
i
=
0
;
i
<
loop
;
++
i
)
{
db_
->
InsertVectors
(
TABLE_NAME
,
nb
,
xb
.
data
(),
vector_ids
);
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
microseconds
(
1
));
}
std
::
vector
<
engine
::
meta
::
DateT
>
dates
;
engine
::
meta
::
DateT
date
;
std
::
string
start_value
=
CurrentTmDate
(
-
3
);
std
::
string
end_value
=
CurrentTmDate
(
-
2
);
ConvertTimeRangeToDBDates
(
start_value
,
end_value
,
dates
);
db_
->
DeleteTable
(
TABLE_NAME
,
dates
);
}
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录