Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
5fdef607
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
5fdef607
编写于
3月 09, 2022
作者:
G
groot
提交者:
GitHub
3月 09, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add rpc interfaces for import (#15930)
Signed-off-by:
N
yhmo
<
yihua.mo@zilliz.com
>
上级
d013134b
变更
27
展开全部
隐藏空白更改
内联
并排
Showing
27 changed file
with
4402 addition
and
864 deletion
+4402
-864
internal/core/src/pb/common.pb.cc
internal/core/src/pb/common.pb.cc
+23
-5
internal/core/src/pb/common.pb.h
internal/core/src/pb/common.pb.h
+32
-0
internal/core/src/pb/milvus.pb.cc
internal/core/src/pb/milvus.pb.cc
+1942
-350
internal/core/src/pb/milvus.pb.h
internal/core/src/pb/milvus.pb.h
+1027
-1
internal/datacoord/mock_test.go
internal/datacoord/mock_test.go
+4
-0
internal/datacoord/services.go
internal/datacoord/services.go
+35
-0
internal/datanode/data_node.go
internal/datanode/data_node.go
+8
-0
internal/distributed/datacoord/client/client.go
internal/distributed/datacoord/client/client.go
+42
-0
internal/distributed/datacoord/service.go
internal/distributed/datacoord/service.go
+15
-0
internal/distributed/datacoord/service_test.go
internal/distributed/datacoord/service_test.go
+14
-0
internal/distributed/datanode/client/client.go
internal/distributed/datanode/client/client.go
+14
-0
internal/distributed/datanode/service.go
internal/distributed/datanode/service.go
+4
-0
internal/distributed/datanode/service_test.go
internal/distributed/datanode/service_test.go
+4
-0
internal/distributed/proxy/service.go
internal/distributed/proxy/service.go
+8
-0
internal/distributed/proxy/service_test.go
internal/distributed/proxy/service_test.go
+20
-0
internal/proto/common.proto
internal/proto/common.proto
+7
-0
internal/proto/commonpb/common.pb.go
internal/proto/commonpb/common.pb.go
+130
-96
internal/proto/data_coord.proto
internal/proto/data_coord.proto
+14
-0
internal/proto/datapb/data_coord.pb.go
internal/proto/datapb/data_coord.pb.go
+393
-183
internal/proto/milvus.proto
internal/proto/milvus.proto
+27
-0
internal/proto/milvuspb/milvus.pb.go
internal/proto/milvuspb/milvus.pb.go
+527
-227
internal/proxy/data_coord_mock_test.go
internal/proxy/data_coord_mock_test.go
+12
-0
internal/proxy/impl.go
internal/proxy/impl.go
+24
-0
internal/types/types.go
internal/types/types.go
+58
-0
internal/util/mock/datacoord_client.go
internal/util/mock/datacoord_client.go
+12
-0
internal/util/mock/datanode_client.go
internal/util/mock/datanode_client.go
+4
-0
scripts/run_go_codecov.sh
scripts/run_go_codecov.sh
+2
-2
未找到文件。
internal/core/src/pb/common.pb.cc
浏览文件 @
5fdef607
...
...
@@ -169,7 +169,7 @@ static void InitDefaultsscc_info_Status_common_2eproto() {
{{
ATOMIC_VAR_INIT
(
::
PROTOBUF_NAMESPACE_ID
::
internal
::
SCCInfoBase
::
kUninitialized
),
0
,
InitDefaultsscc_info_Status_common_2eproto
},
{}};
static
::
PROTOBUF_NAMESPACE_ID
::
Metadata
file_level_metadata_common_2eproto
[
8
];
static
const
::
PROTOBUF_NAMESPACE_ID
::
EnumDescriptor
*
file_level_enum_descriptors_common_2eproto
[
7
];
static
const
::
PROTOBUF_NAMESPACE_ID
::
EnumDescriptor
*
file_level_enum_descriptors_common_2eproto
[
8
];
static
constexpr
::
PROTOBUF_NAMESPACE_ID
::
ServiceDescriptor
const
**
file_level_service_descriptors_common_2eproto
=
nullptr
;
const
::
PROTOBUF_NAMESPACE_ID
::
uint32
TableStruct_common_2eproto
::
offsets
[]
PROTOBUF_SECTION_VARIABLE
(
protodesc_cold
)
=
{
...
...
@@ -319,9 +319,11 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"
\n\017
CompactionState
\022\021\n\r
UndefiedState
\020\000\022\r\n\t
"
"Executing
\020\001\022\r\n\t
Completed
\020\002
*X
\n\020
Consistenc"
"yLevel
\022\n\n\006
Strong
\020\000\022\013\n\007
Session
\020\001\022\013\n\007
Bound"
"ed
\020\002\022\016\n\n
Eventually
\020\003\022\016\n\n
Customized
\020\004
B5Z3"
"github.com/milvus-io/milvus/internal/pro"
"to/commonpbb
\006
proto3"
"ed
\020\002\022\016\n\n
Eventually
\020\003\022\016\n\n
Customized
\020\004
*
\\\n\013
"
"ImportState
\022\021\n\r
ImportPending
\020\000\022\023\n\017
Import"
"Executing
\020\001\022\023\n\017
ImportCompleted
\020\002\022\020\n\014
Impo"
"rtFailed
\020\003
B5Z3github.com/milvus-io/milvu"
"s/internal/proto/commonpbb
\006
proto3"
;
static
const
::
PROTOBUF_NAMESPACE_ID
::
internal
::
DescriptorTable
*
const
descriptor_table_common_2eproto_deps
[
1
]
=
{
};
...
...
@@ -338,7 +340,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_com
static
::
PROTOBUF_NAMESPACE_ID
::
internal
::
once_flag
descriptor_table_common_2eproto_once
;
static
bool
descriptor_table_common_2eproto_initialized
=
false
;
const
::
PROTOBUF_NAMESPACE_ID
::
internal
::
DescriptorTable
descriptor_table_common_2eproto
=
{
&
descriptor_table_common_2eproto_initialized
,
descriptor_table_protodef_common_2eproto
,
"common.proto"
,
2
739
,
&
descriptor_table_common_2eproto_initialized
,
descriptor_table_protodef_common_2eproto
,
"common.proto"
,
2
833
,
&
descriptor_table_common_2eproto_once
,
descriptor_table_common_2eproto_sccs
,
descriptor_table_common_2eproto_deps
,
8
,
0
,
schemas
,
file_default_instances
,
TableStruct_common_2eproto
::
offsets
,
file_level_metadata_common_2eproto
,
8
,
file_level_enum_descriptors_common_2eproto
,
file_level_service_descriptors_common_2eproto
,
...
...
@@ -540,6 +542,22 @@ bool ConsistencyLevel_IsValid(int value) {
}
}
const
::
PROTOBUF_NAMESPACE_ID
::
EnumDescriptor
*
ImportState_descriptor
()
{
::
PROTOBUF_NAMESPACE_ID
::
internal
::
AssignDescriptors
(
&
descriptor_table_common_2eproto
);
return
file_level_enum_descriptors_common_2eproto
[
7
];
}
bool
ImportState_IsValid
(
int
value
)
{
switch
(
value
)
{
case
0
:
case
1
:
case
2
:
case
3
:
return
true
;
default:
return
false
;
}
}
// ===================================================================
...
...
internal/core/src/pb/common.pb.h
浏览文件 @
5fdef607
...
...
@@ -367,6 +367,33 @@ inline bool ConsistencyLevel_Parse(
return
::
PROTOBUF_NAMESPACE_ID
::
internal
::
ParseNamedEnum
<
ConsistencyLevel
>
(
ConsistencyLevel_descriptor
(),
name
,
value
);
}
enum
ImportState
:
int
{
ImportPending
=
0
,
ImportExecuting
=
1
,
ImportCompleted
=
2
,
ImportFailed
=
3
,
ImportState_INT_MIN_SENTINEL_DO_NOT_USE_
=
std
::
numeric_limits
<::
PROTOBUF_NAMESPACE_ID
::
int32
>::
min
(),
ImportState_INT_MAX_SENTINEL_DO_NOT_USE_
=
std
::
numeric_limits
<::
PROTOBUF_NAMESPACE_ID
::
int32
>::
max
()
};
bool
ImportState_IsValid
(
int
value
);
constexpr
ImportState
ImportState_MIN
=
ImportPending
;
constexpr
ImportState
ImportState_MAX
=
ImportFailed
;
constexpr
int
ImportState_ARRAYSIZE
=
ImportState_MAX
+
1
;
const
::
PROTOBUF_NAMESPACE_ID
::
EnumDescriptor
*
ImportState_descriptor
();
template
<
typename
T
>
inline
const
std
::
string
&
ImportState_Name
(
T
enum_t_value
)
{
static_assert
(
::
std
::
is_same
<
T
,
ImportState
>::
value
||
::
std
::
is_integral
<
T
>::
value
,
"Incorrect type passed to function ImportState_Name."
);
return
::
PROTOBUF_NAMESPACE_ID
::
internal
::
NameOfEnum
(
ImportState_descriptor
(),
enum_t_value
);
}
inline
bool
ImportState_Parse
(
const
std
::
string
&
name
,
ImportState
*
value
)
{
return
::
PROTOBUF_NAMESPACE_ID
::
internal
::
ParseNamedEnum
<
ImportState
>
(
ImportState_descriptor
(),
name
,
value
);
}
// ===================================================================
class
Status
:
...
...
@@ -2219,6 +2246,11 @@ template <>
inline
const
EnumDescriptor
*
GetEnumDescriptor
<
::
milvus
::
proto
::
common
::
ConsistencyLevel
>
()
{
return
::
milvus
::
proto
::
common
::
ConsistencyLevel_descriptor
();
}
template
<
>
struct
is_proto_enum
<
::
milvus
::
proto
::
common
::
ImportState
>
:
::
std
::
true_type
{};
template
<
>
inline
const
EnumDescriptor
*
GetEnumDescriptor
<
::
milvus
::
proto
::
common
::
ImportState
>
()
{
return
::
milvus
::
proto
::
common
::
ImportState_descriptor
();
}
PROTOBUF_NAMESPACE_CLOSE
...
...
internal/core/src/pb/milvus.pb.cc
浏览文件 @
5fdef607
此差异已折叠。
点击以展开。
internal/core/src/pb/milvus.pb.h
浏览文件 @
5fdef607
此差异已折叠。
点击以展开。
internal/datacoord/mock_test.go
浏览文件 @
5fdef607
...
...
@@ -196,6 +196,10 @@ func (c *mockDataNodeClient) Compaction(ctx context.Context, req *datapb.Compact
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
Reason
:
"not implemented"
},
nil
}
func
(
c
*
mockDataNodeClient
)
Import
(
ctx
context
.
Context
,
in
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_Success
},
nil
}
func
(
c
*
mockDataNodeClient
)
Stop
()
error
{
c
.
state
=
internalpb
.
StateCode_Abnormal
return
nil
...
...
internal/datacoord/services.go
浏览文件 @
5fdef607
...
...
@@ -961,3 +961,38 @@ func (s *Server) GetFlushState(ctx context.Context, req *milvuspb.GetFlushStateR
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_Success
return
resp
,
nil
}
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
func
(
s
*
Server
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
log
.
Info
(
"receive import request"
)
resp
:=
&
milvuspb
.
ImportResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
},
}
return
resp
,
nil
}
// Check import task state from datanode
func
(
s
*
Server
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
log
.
Info
(
"receive get import state request"
)
resp
:=
&
milvuspb
.
GetImportStateResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
},
}
return
resp
,
nil
}
// Report impot task state to datacoord
func
(
s
*
Server
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
log
.
Info
(
"receive complete import request"
)
resp
:=
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
}
return
resp
,
nil
}
internal/datanode/data_node.go
浏览文件 @
5fdef607
...
...
@@ -777,3 +777,11 @@ func (node *DataNode) Compaction(ctx context.Context, req *datapb.CompactionPlan
ErrorCode
:
commonpb
.
ErrorCode_Success
,
},
nil
}
// Compaction handles compaction request from DataCoord
// returns status as long as compaction task enqueued or invalid
func
(
node
*
DataNode
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_UnexpectedError
,
},
nil
}
internal/distributed/datacoord/client/client.go
浏览文件 @
5fdef607
...
...
@@ -486,3 +486,45 @@ func (c *Client) DropVirtualChannel(ctx context.Context, req *datapb.DropVirtual
}
return
ret
.
(
*
datapb
.
DropVirtualChannelResponse
),
err
}
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
func
(
c
*
Client
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
ret
,
err
:=
c
.
grpcClient
.
ReCall
(
ctx
,
func
(
client
interface
{})
(
interface
{},
error
)
{
if
!
funcutil
.
CheckCtxValid
(
ctx
)
{
return
nil
,
ctx
.
Err
()
}
return
client
.
(
datapb
.
DataCoordClient
)
.
Import
(
ctx
,
req
)
})
if
err
!=
nil
||
ret
==
nil
{
return
nil
,
err
}
return
ret
.
(
*
milvuspb
.
ImportResponse
),
err
}
// Check import task state from datanode
func
(
c
*
Client
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
ret
,
err
:=
c
.
grpcClient
.
ReCall
(
ctx
,
func
(
client
interface
{})
(
interface
{},
error
)
{
if
!
funcutil
.
CheckCtxValid
(
ctx
)
{
return
nil
,
ctx
.
Err
()
}
return
client
.
(
datapb
.
DataCoordClient
)
.
GetImportState
(
ctx
,
req
)
})
if
err
!=
nil
||
ret
==
nil
{
return
nil
,
err
}
return
ret
.
(
*
milvuspb
.
GetImportStateResponse
),
err
}
// Report impot task state to datacoord
func
(
c
*
Client
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
ret
,
err
:=
c
.
grpcClient
.
ReCall
(
ctx
,
func
(
client
interface
{})
(
interface
{},
error
)
{
if
!
funcutil
.
CheckCtxValid
(
ctx
)
{
return
nil
,
ctx
.
Err
()
}
return
client
.
(
datapb
.
DataCoordClient
)
.
CompleteImport
(
ctx
,
req
)
})
if
err
!=
nil
||
ret
==
nil
{
return
nil
,
err
}
return
ret
.
(
*
commonpb
.
Status
),
err
}
internal/distributed/datacoord/service.go
浏览文件 @
5fdef607
...
...
@@ -327,3 +327,18 @@ func (s *Server) GetFlushState(ctx context.Context, req *milvuspb.GetFlushStateR
func
(
s
*
Server
)
DropVirtualChannel
(
ctx
context
.
Context
,
req
*
datapb
.
DropVirtualChannelRequest
)
(
*
datapb
.
DropVirtualChannelResponse
,
error
)
{
return
s
.
dataCoord
.
DropVirtualChannel
(
ctx
,
req
)
}
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
func
(
s
*
Server
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
s
.
dataCoord
.
Import
(
ctx
,
req
)
}
// Check import task state from datanode
func
(
s
*
Server
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
s
.
dataCoord
.
GetImportState
(
ctx
,
req
)
}
// Report impot task state to datacoord
func
(
s
*
Server
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
return
s
.
dataCoord
.
CompleteImport
(
ctx
,
req
)
}
internal/distributed/datacoord/service_test.go
浏览文件 @
5fdef607
...
...
@@ -55,6 +55,8 @@ type MockDataCoord struct {
watchChannelsResp
*
datapb
.
WatchChannelsResponse
getFlushStateResp
*
milvuspb
.
GetFlushStateResponse
dropVChanResp
*
datapb
.
DropVirtualChannelResponse
importResp
*
milvuspb
.
ImportResponse
getImportStateResp
*
milvuspb
.
GetImportStateResponse
}
func
(
m
*
MockDataCoord
)
Init
()
error
{
...
...
@@ -164,6 +166,18 @@ func (m *MockDataCoord) DropVirtualChannel(ctx context.Context, req *datapb.Drop
return
m
.
dropVChanResp
,
m
.
err
}
func
(
m
*
MockDataCoord
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
m
.
importResp
,
m
.
err
}
func
(
m
*
MockDataCoord
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
m
.
getImportStateResp
,
m
.
err
}
func
(
m
*
MockDataCoord
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
return
m
.
status
,
m
.
err
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func
Test_NewServer
(
t
*
testing
.
T
)
{
ctx
:=
context
.
Background
()
...
...
internal/distributed/datanode/client/client.go
浏览文件 @
5fdef607
...
...
@@ -181,3 +181,17 @@ func (c *Client) Compaction(ctx context.Context, req *datapb.CompactionPlan) (*c
}
return
ret
.
(
*
commonpb
.
Status
),
err
}
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
func
(
c
*
Client
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
{
ret
,
err
:=
c
.
grpcClient
.
ReCall
(
ctx
,
func
(
client
interface
{})
(
interface
{},
error
)
{
if
!
funcutil
.
CheckCtxValid
(
ctx
)
{
return
nil
,
ctx
.
Err
()
}
return
client
.
(
datapb
.
DataNodeClient
)
.
Import
(
ctx
,
req
)
})
if
err
!=
nil
||
ret
==
nil
{
return
nil
,
err
}
return
ret
.
(
*
commonpb
.
Status
),
err
}
internal/distributed/datanode/service.go
浏览文件 @
5fdef607
...
...
@@ -353,3 +353,7 @@ func (s *Server) GetMetrics(ctx context.Context, request *milvuspb.GetMetricsReq
func
(
s
*
Server
)
Compaction
(
ctx
context
.
Context
,
request
*
datapb
.
CompactionPlan
)
(
*
commonpb
.
Status
,
error
)
{
return
s
.
datanode
.
Compaction
(
ctx
,
request
)
}
func
(
s
*
Server
)
Import
(
ctx
context
.
Context
,
request
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
s
.
datanode
.
Import
(
ctx
,
request
)
}
internal/distributed/datanode/service_test.go
浏览文件 @
5fdef607
...
...
@@ -111,6 +111,10 @@ func (m *MockDataNode) Compaction(ctx context.Context, req *datapb.CompactionPla
func
(
m
*
MockDataNode
)
SetEtcdClient
(
client
*
clientv3
.
Client
)
{
}
func
(
m
*
MockDataNode
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
{
return
m
.
status
,
m
.
err
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
type
mockDataCoord
struct
{
types
.
DataCoord
...
...
internal/distributed/proxy/service.go
浏览文件 @
5fdef607
...
...
@@ -630,3 +630,11 @@ func (s *Server) SendSearchResult(ctx context.Context, results *internalpb.Searc
func
(
s
*
Server
)
SendRetrieveResult
(
ctx
context
.
Context
,
results
*
internalpb
.
RetrieveResults
)
(
*
commonpb
.
Status
,
error
)
{
return
s
.
proxy
.
SendRetrieveResult
(
ctx
,
results
)
}
func
(
s
*
Server
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
s
.
proxy
.
Import
(
ctx
,
req
)
}
func
(
s
*
Server
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
s
.
proxy
.
GetImportState
(
ctx
,
req
)
}
internal/distributed/proxy/service_test.go
浏览文件 @
5fdef607
...
...
@@ -419,6 +419,18 @@ func (m *MockDataCoord) DropVirtualChannel(ctx context.Context, req *datapb.Drop
return
&
datapb
.
DropVirtualChannelResponse
{},
nil
}
func
(
m
*
MockDataCoord
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
nil
,
nil
}
func
(
m
*
MockDataCoord
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
nil
,
nil
}
func
(
m
*
MockDataCoord
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
return
nil
,
nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
type
MockProxy
struct
{
MockBase
...
...
@@ -644,6 +656,14 @@ func (m *MockProxy) SendRetrieveResult(ctx context.Context, req *internalpb.Retr
return
nil
,
nil
}
func
(
m
*
MockProxy
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
nil
,
nil
}
func
(
m
*
MockProxy
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
nil
,
nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func
Test_NewServer
(
t
*
testing
.
T
)
{
ctx
:=
context
.
Background
()
...
...
internal/proto/common.proto
浏览文件 @
5fdef607
...
...
@@ -192,3 +192,10 @@ enum ConsistencyLevel {
Eventually
=
3
;
Customized
=
4
;
// Users pass their own `guarantee_timestamp`.
}
enum
ImportState
{
ImportPending
=
0
;
ImportExecuting
=
1
;
ImportCompleted
=
2
;
ImportFailed
=
3
;
}
\ No newline at end of file
internal/proto/commonpb/common.pb.go
浏览文件 @
5fdef607
此差异已折叠。
点击以展开。
internal/proto/data_coord.proto
浏览文件 @
5fdef607
...
...
@@ -41,6 +41,11 @@ service DataCoord {
rpc
WatchChannels
(
WatchChannelsRequest
)
returns
(
WatchChannelsResponse
)
{}
rpc
GetFlushState
(
milvus.GetFlushStateRequest
)
returns
(
milvus.GetFlushStateResponse
)
{}
rpc
DropVirtualChannel
(
DropVirtualChannelRequest
)
returns
(
DropVirtualChannelResponse
)
{}
// https://wiki.lfaidata.foundation/display/MIL/MEP+24+--+Support+bulk+load
rpc
Import
(
milvus.ImportRequest
)
returns
(
milvus.ImportResponse
)
{}
rpc
GetImportState
(
milvus.GetImportStateRequest
)
returns
(
milvus.GetImportStateResponse
)
{}
rpc
CompleteImport
(
ImportResult
)
returns
(
common.Status
)
{}
}
service
DataNode
{
...
...
@@ -53,6 +58,9 @@ service DataNode {
// https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy
rpc
GetMetrics
(
milvus.GetMetricsRequest
)
returns
(
milvus.GetMetricsResponse
)
{}
rpc
Compaction
(
CompactionPlan
)
returns
(
common.Status
)
{}
// https://wiki.lfaidata.foundation/display/MIL/MEP+24+--+Support+bulk+load
rpc
Import
(
milvus.ImportRequest
)
returns
(
common.Status
)
{}
}
message
FlushRequest
{
...
...
@@ -411,3 +419,9 @@ message DropVirtualChannelSegment {
message
DropVirtualChannelResponse
{
common.Status
status
=
1
;
}
message
ImportResult
{
common.Status
status
=
1
;
repeated
int64
segments
=
2
;
// id array of new sealed segments
int64
row_count
=
3
;
// how many rows are imported by this task
}
\ No newline at end of file
internal/proto/datapb/data_coord.pb.go
浏览文件 @
5fdef607
此差异已折叠。
点击以展开。
internal/proto/milvus.proto
浏览文件 @
5fdef607
...
...
@@ -56,6 +56,10 @@ service MilvusService {
rpc
GetCompactionState
(
GetCompactionStateRequest
)
returns
(
GetCompactionStateResponse
)
{}
rpc
ManualCompaction
(
ManualCompactionRequest
)
returns
(
ManualCompactionResponse
)
{}
rpc
GetCompactionStateWithPlans
(
GetCompactionPlansRequest
)
returns
(
GetCompactionPlansResponse
)
{}
// https://wiki.lfaidata.foundation/display/MIL/MEP+24+--+Support+bulk+load
rpc
Import
(
ImportRequest
)
returns
(
ImportResponse
)
{}
rpc
GetImportState
(
GetImportStateRequest
)
returns
(
GetImportStateResponse
)
{}
}
message
CreateAliasRequest
{
...
...
@@ -783,6 +787,29 @@ message GetFlushStateResponse {
bool
flushed
=
2
;
}
message
ImportRequest
{
string
collection_name
=
1
;
// target collection
string
partition_name
=
2
;
// target partition
bool
row_based
=
3
;
// the file is row-based or column-based
repeated
string
files
=
4
;
// file paths to be imported
repeated
common.KeyValuePair
options
=
5
;
// import options
}
message
ImportResponse
{
common.Status
status
=
1
;
repeated
int64
tasks
=
2
;
// id array of import tasks
}
message
GetImportStateRequest
{
int64
task
=
1
;
// id of an import task
}
message
GetImportStateResponse
{
common.Status
status
=
1
;
common.ImportState
state
=
2
;
// is this import task finished or not
int64
row_count
=
3
;
// if the task is finished, this value is how many rows are imported. if the task is not finished, this value is how many rows are parsed.
}
service
ProxyService
{
rpc
RegisterLink
(
RegisterLinkRequest
)
returns
(
RegisterLinkResponse
)
{}
}
internal/proto/milvuspb/milvus.pb.go
浏览文件 @
5fdef607
此差异已折叠。
点击以展开。
internal/proxy/data_coord_mock_test.go
浏览文件 @
5fdef607
...
...
@@ -206,6 +206,18 @@ func (coord *DataCoordMock) DropVirtualChannel(ctx context.Context, req *datapb.
return
&
datapb
.
DropVirtualChannelResponse
{},
nil
}
func
(
coord
*
DataCoordMock
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
&
milvuspb
.
ImportResponse
{},
nil
}
func
(
coord
*
DataCoordMock
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
&
milvuspb
.
GetImportStateResponse
{},
nil
}
func
(
coord
*
DataCoordMock
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{},
nil
}
func
NewDataCoordMock
()
*
DataCoordMock
{
return
&
DataCoordMock
{
nodeID
:
typeutil
.
UniqueID
(
uniquegenerator
.
GetUniqueIntGeneratorIns
()
.
GetInt
()),
...
...
internal/proxy/impl.go
浏览文件 @
5fdef607
...
...
@@ -3939,3 +3939,27 @@ func unhealthyStatus() *commonpb.Status {
Reason
:
"proxy not healthy"
,
}
}
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
func
(
node
*
Proxy
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
log
.
Info
(
"received Import request"
)
resp
:=
&
milvuspb
.
ImportResponse
{}
if
!
node
.
checkHealthy
()
{
resp
.
Status
=
unhealthyStatus
()
return
resp
,
nil
}
return
resp
,
nil
}
// Check import task state from datanode
func
(
node
*
Proxy
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
log
.
Info
(
"received GetImportState request"
,
zap
.
Int64
(
"taskID"
,
req
.
GetTask
()))
resp
:=
&
milvuspb
.
GetImportStateResponse
{}
if
!
node
.
checkHealthy
()
{
resp
.
Status
=
unhealthyStatus
()
return
resp
,
nil
}
return
resp
,
nil
}
internal/types/types.go
浏览文件 @
5fdef607
...
...
@@ -68,6 +68,15 @@ type DataNode interface {
GetMetrics
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetMetricsRequest
)
(
*
milvuspb
.
GetMetricsResponse
,
error
)
// Compaction will add a compaction task according to the request plan
Compaction
(
ctx
context
.
Context
,
req
*
datapb
.
CompactionPlan
)
(
*
commonpb
.
Status
,
error
)
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
//
// ctx is the context to control request deadline and cancellation
// req contains the request params, including file path and options
//
// Return status indicates if this operation is processed successfully or fail cause;
// error is always nil
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
commonpb
.
Status
,
error
)
}
// DataNodeComponent is used by grpc server of DataNode
...
...
@@ -256,6 +265,35 @@ type DataCoord interface {
// response status contains the status/error code and failing reason if any
// error is returned only when some communication issue occurs
DropVirtualChannel
(
ctx
context
.
Context
,
req
*
datapb
.
DropVirtualChannelRequest
)
(
*
datapb
.
DropVirtualChannelResponse
,
error
)
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
//
// ctx is the context to control request deadline and cancellation
// req contains the request params, including file path and options
//
// The `Status` in response struct `ImportResponse` indicates if this operation is processed successfully or fail cause;
// the `tasks` in `ImportResponse` return an id list of tasks.
// error is always nil
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
// Check import task state from datanode
//
// ctx is the context to control request deadline and cancellation
// req contains the request params, including a task id
//
// The `Status` in response struct `GetImportStateResponse` indicates if this operation is processed successfully or fail cause;
// the `state` in `GetImportStateResponse` return the state of the import task.
// error is always nil
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
// Report impot task state to datacoord
//
// ctx is the context to control request deadline and cancellation
// req contains the import results, including imported row count and an id list of generated segments
//
// response status contains the status/error code and failing reason if any error is returned
// error is always nil
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
)
(
*
commonpb
.
Status
,
error
)
}
// DataCoordComponent defines the interface of DataCoord component.
...
...
@@ -1030,6 +1068,26 @@ type ProxyComponent interface {
GetCompactionStateWithPlans
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetCompactionPlansRequest
)
(
*
milvuspb
.
GetCompactionPlansResponse
,
error
)
// GetFlushState gets the flush state of multiple segments
GetFlushState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetFlushStateRequest
)
(
*
milvuspb
.
GetFlushStateResponse
,
error
)
// Import data files(json, numpy, etc.) on MinIO/S3 storage, read and parse them into sealed segments
//
// ctx is the context to control request deadline and cancellation
// req contains the request params, including file path and options
//
// The `Status` in response struct `ImportResponse` indicates if this operation is processed successfully or fail cause;
// the `tasks` in `ImportResponse` return an id list of tasks.
// error is always nil
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
)
(
*
milvuspb
.
ImportResponse
,
error
)
// Check import task state from datanode
//
// ctx is the context to control request deadline and cancellation
// req contains the request params, including a task id
//
// The `Status` in response struct `GetImportStateResponse` indicates if this operation is processed successfully or fail cause;
// the `state` in `GetImportStateResponse` return the state of the import task.
// error is always nil
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
}
// QueryNode is the interface `querynode` package implements
...
...
internal/util/mock/datacoord_client.go
浏览文件 @
5fdef607
...
...
@@ -118,3 +118,15 @@ func (m *DataCoordClient) GetFlushState(ctx context.Context, req *milvuspb.GetFl
func
(
m
*
DataCoordClient
)
DropVirtualChannel
(
ctx
context
.
Context
,
req
*
datapb
.
DropVirtualChannelRequest
,
opts
...
grpc
.
CallOption
)
(
*
datapb
.
DropVirtualChannelResponse
,
error
)
{
return
&
datapb
.
DropVirtualChannelResponse
{},
m
.
Err
}
func
(
m
*
DataCoordClient
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
,
opts
...
grpc
.
CallOption
)
(
*
milvuspb
.
ImportResponse
,
error
)
{
return
&
milvuspb
.
ImportResponse
{},
m
.
Err
}
func
(
m
*
DataCoordClient
)
GetImportState
(
ctx
context
.
Context
,
req
*
milvuspb
.
GetImportStateRequest
,
opts
...
grpc
.
CallOption
)
(
*
milvuspb
.
GetImportStateResponse
,
error
)
{
return
&
milvuspb
.
GetImportStateResponse
{},
m
.
Err
}
func
(
m
*
DataCoordClient
)
CompleteImport
(
ctx
context
.
Context
,
req
*
datapb
.
ImportResult
,
opts
...
grpc
.
CallOption
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{},
m
.
Err
}
internal/util/mock/datanode_client.go
浏览文件 @
5fdef607
...
...
@@ -54,3 +54,7 @@ func (m *DataNodeClient) GetMetrics(ctx context.Context, in *milvuspb.GetMetrics
func
(
m
*
DataNodeClient
)
Compaction
(
ctx
context
.
Context
,
req
*
datapb
.
CompactionPlan
,
opts
...
grpc
.
CallOption
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{},
m
.
Err
}
func
(
m
*
DataNodeClient
)
Import
(
ctx
context
.
Context
,
req
*
milvuspb
.
ImportRequest
,
opts
...
grpc
.
CallOption
)
(
*
commonpb
.
Status
,
error
)
{
return
&
commonpb
.
Status
{},
m
.
Err
}
scripts/run_go_codecov.sh
浏览文件 @
5fdef607
...
...
@@ -27,7 +27,7 @@ echo "mode: atomic" > ${FILE_COVERAGE_INFO}
echo
"Running unittest under ./internal"
if
[[
"
$(
uname
-s
)
"
==
"Darwin"
]]
;
then
export
MallocNanoZone
=
0
for
d
in
$(
go list ./internal... |
grep
-v
-e
vendor
-e
internal/querycoord
-e
/metricsinfo
-e
internal/proxy
-e
internal/querynode
)
;
do
for
d
in
$(
go list ./internal
/
... |
grep
-v
-e
vendor
-e
internal/querycoord
-e
/metricsinfo
-e
internal/proxy
-e
internal/querynode
)
;
do
go
test
-race
-v
-coverpkg
=
./...
-coverprofile
=
profile.out
-covermode
=
atomic
"
$d
"
if
[
-f
profile.out
]
;
then
sed
'1d'
profile.out
>>
${
FILE_COVERAGE_INFO
}
...
...
@@ -35,7 +35,7 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
fi
done
else
for
d
in
$(
go list ./internal... |
grep
-v
vendor
)
;
do
for
d
in
$(
go list ./internal
/
... |
grep
-v
vendor
)
;
do
go
test
-race
-v
-coverpkg
=
./...
-coverprofile
=
profile.out
-covermode
=
atomic
"
$d
"
if
[
-f
profile.out
]
;
then
sed
'1d'
profile.out
>>
${
FILE_COVERAGE_INFO
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录