Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
e83ac41b
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
e83ac41b
编写于
1月 26, 2021
作者:
Z
zhenshan.cao
提交者:
yefu.chen
1月 26, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add component for index
Signed-off-by:
N
zhenshan.cao
<
zhenshan.cao@zilliz.com
>
上级
8c54af29
变更
9
展开全部
隐藏空白更改
内联
并排
Showing
9 changed file
with
454 addition
and
70 deletion
+454
-70
Makefile
Makefile
+12
-5
internal/distributed/indexnode/client/client.go
internal/distributed/indexnode/client/client.go
+30
-0
internal/distributed/indexnode/service.go
internal/distributed/indexnode/service.go
+38
-0
internal/distributed/indexservice/service.go
internal/distributed/indexservice/service.go
+40
-13
internal/indexnode/indexnode.go
internal/indexnode/indexnode.go
+29
-0
internal/indexservice/indexservice.go
internal/indexservice/indexservice.go
+25
-3
internal/proto/index_service.proto
internal/proto/index_service.proto
+8
-0
internal/proto/indexpb/index_service.pb.go
internal/proto/indexpb/index_service.pb.go
+271
-49
internal/util/typeutil/interface.go
internal/util/typeutil/interface.go
+1
-0
未找到文件。
Makefile
浏览文件 @
e83ac41b
...
...
@@ -80,13 +80,14 @@ verifiers: getdeps cppcheck fmt static-check ruleguard
master
:
build-cpp
@
echo
"Building each component's binary to './bin'"
@
echo
"Building master ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"0"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/master
$(PWD)
/cmd/master/main.go 1>/dev/null
@
echo
"Building masterservice ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"1"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/masterservice
$(PWD)
/cmd/masterservice/main.go 1>/dev/null
# Builds various components locally.
proxynode
:
build-cpp
@
echo
"Building each component's binary to './bin'"
@
echo
"Building proxy ..."
@
echo
"Building proxy
node
..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"0"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/proxynode
$(PWD)
/cmd/proxy/node/proxy_node.go 1>/dev/null
# Builds various components locally.
...
...
@@ -95,6 +96,7 @@ querynode: build-cpp
@
echo
"Building query node ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"1"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/querynode
$(PWD)
/cmd/querynode/query_node.go 1>/dev/null
# Builds various components locally.
writenode
:
build-cpp
@
echo
"Building each component's binary to './bin'"
...
...
@@ -110,8 +112,13 @@ datanode: build-cpp
# Builds various components locally.
indexnode
:
build-cpp
@
echo
"Building each component's binary to './bin'"
@
echo
"Building indexbuilder ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"1"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/indexbuilder
$(PWD)
/cmd/indexbuilder/indexbuilder.go 1>/dev/null
@
echo
"Building distributed indexnode ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"1"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/distributed/indexnode
$(PWD)
/cmd/distributed/indexnode/main.go 1>/dev/null
indexservice
:
build-cpp
@
echo
"Building each component's binary to './bin'"
@
echo
"Building distributed indexservice ..."
@
mkdir
-p
$(INSTALL_PATH)
&&
go
env
-w
CGO_ENABLED
=
"1"
&&
GO111MODULE
=
on
$(GO)
build
-o
$(INSTALL_PATH)
/distributed/indexservice
$(PWD)
/cmd/distributed/indexservice/main.go 1>/dev/null
# Builds various components locally.
...
...
internal/distributed/indexnode/client/client.go
浏览文件 @
e83ac41b
...
...
@@ -2,11 +2,13 @@ package grpcindexnodeclient
import
(
"context"
"errors"
"log"
"time"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"google.golang.org/grpc"
)
...
...
@@ -15,6 +17,34 @@ type Client struct {
nodeAddress
string
}
func
(
c
Client
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
return
c
.
grpcClient
.
GetComponentStates
(
context
.
Background
(),
&
commonpb
.
Empty
{})
}
func
(
c
Client
)
GetTimeTickChannel
()
(
string
,
error
)
{
resp
,
err
:=
c
.
grpcClient
.
GetTimeTickChannel
(
context
.
Background
(),
&
commonpb
.
Empty
{})
if
err
!=
nil
{
return
""
,
err
}
if
resp
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_SUCCESS
{
return
""
,
errors
.
New
(
resp
.
Status
.
Reason
)
}
return
resp
.
Value
,
nil
}
func
(
c
Client
)
GetStatisticsChannel
()
(
string
,
error
)
{
resp
,
err
:=
c
.
grpcClient
.
GetStatisticsChannel
(
context
.
Background
(),
&
commonpb
.
Empty
{})
if
err
!=
nil
{
return
""
,
err
}
if
resp
.
Status
.
ErrorCode
!=
commonpb
.
ErrorCode_SUCCESS
{
return
""
,
errors
.
New
(
resp
.
Status
.
Reason
)
}
return
resp
.
Value
,
nil
}
func
(
c
Client
)
Init
()
error
{
return
nil
}
...
...
internal/distributed/indexnode/service.go
浏览文件 @
e83ac41b
...
...
@@ -11,6 +11,8 @@ import (
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
"google.golang.org/grpc"
)
...
...
@@ -25,6 +27,42 @@ type Server struct {
loopWg
sync
.
WaitGroup
}
func
(
s
*
Server
)
GetComponentStates
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
internalpb2
.
ComponentStates
,
error
)
{
return
s
.
node
.
GetComponentStates
()
}
func
(
s
*
Server
)
GetTimeTickChannel
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
milvuspb
.
StringResponse
,
error
)
{
ret
,
err
:=
s
.
node
.
GetTimeTickChannel
()
resp
:=
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
if
err
!=
nil
{
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_UNEXPECTED_ERROR
resp
.
Status
.
Reason
=
err
.
Error
()
}
else
{
resp
.
Value
=
ret
}
return
resp
,
nil
}
func
(
s
*
Server
)
GetStatisticsChannel
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
milvuspb
.
StringResponse
,
error
)
{
ret
,
err
:=
s
.
node
.
GetStatisticsChannel
()
resp
:=
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
if
err
!=
nil
{
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_UNEXPECTED_ERROR
resp
.
Status
.
Reason
=
err
.
Error
()
}
else
{
resp
.
Value
=
ret
}
return
resp
,
nil
}
func
(
s
*
Server
)
registerNode
()
error
{
log
.
Printf
(
"Registering node. IP = %s, Port = %d"
,
indexnode
.
Params
.
NodeIP
,
indexnode
.
Params
.
NodePort
)
...
...
internal/distributed/indexservice/service.go
浏览文件 @
e83ac41b
...
...
@@ -7,10 +7,12 @@ import (
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
"github.com/zilliztech/milvus-distributed/internal/indexservice"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
"google.golang.org/grpc"
)
...
...
@@ -28,6 +30,43 @@ type Server struct {
loopWg
sync
.
WaitGroup
}
func
(
s
*
Server
)
GetComponentStates
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
internalpb2
.
ComponentStates
,
error
)
{
return
s
.
server
.
GetComponentStates
()
}
func
(
s
*
Server
)
GetTimeTickChannel
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
milvuspb
.
StringResponse
,
error
)
{
resp
:=
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
channel
,
err
:=
s
.
server
.
GetTimeTickChannel
()
if
err
!=
nil
{
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_UNEXPECTED_ERROR
resp
.
Status
.
Reason
=
err
.
Error
()
return
resp
,
nil
}
resp
.
Value
=
channel
return
resp
,
nil
}
func
(
s
*
Server
)
GetStatisticsChannel
(
ctx
context
.
Context
,
empty
*
commonpb
.
Empty
)
(
*
milvuspb
.
StringResponse
,
error
)
{
resp
:=
&
milvuspb
.
StringResponse
{
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
channel
,
err
:=
s
.
server
.
GetStatisticsChannel
()
if
err
!=
nil
{
resp
.
Status
.
ErrorCode
=
commonpb
.
ErrorCode_UNEXPECTED_ERROR
resp
.
Status
.
Reason
=
err
.
Error
()
return
resp
,
nil
}
resp
.
Value
=
channel
return
resp
,
nil
}
func
Init
()
error
{
indexservice
.
Params
.
Init
()
return
nil
...
...
@@ -48,18 +87,6 @@ func (s *Server) Stop() error {
return
nil
}
func
(
s
*
Server
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
panic
(
"implement me"
)
}
func
(
s
*
Server
)
GetTimeTickChannel
()
(
string
,
error
)
{
panic
(
"implement me"
)
}
func
(
s
*
Server
)
GetStatisticsChannel
()
(
string
,
error
)
{
panic
(
"implement me"
)
}
func
(
s
*
Server
)
RegisterNode
(
ctx
context
.
Context
,
req
*
indexpb
.
RegisterNodeRequest
)
(
*
indexpb
.
RegisterNodeResponse
,
error
)
{
return
s
.
server
.
RegisterNode
(
req
)
...
...
internal/indexnode/indexnode.go
浏览文件 @
e83ac41b
...
...
@@ -11,6 +11,7 @@ import (
miniokv
"github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/util/retry"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
...
...
@@ -23,6 +24,8 @@ type UniqueID = typeutil.UniqueID
type
Timestamp
=
typeutil
.
Timestamp
type
IndexNode
struct
{
state
internalpb2
.
StateCode
loopCtx
context
.
Context
loopCancel
func
()
...
...
@@ -162,3 +165,29 @@ func (i *IndexNode) BuildIndex(request *indexpb.BuildIndexCmd) (*commonpb.Status
}
return
ret
,
nil
}
func
(
i
*
IndexNode
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
stateInfo
:=
&
internalpb2
.
ComponentInfo
{
NodeID
:
Params
.
NodeID
,
Role
:
"IndexNode"
,
StateCode
:
i
.
state
,
}
ret
:=
&
internalpb2
.
ComponentStates
{
State
:
stateInfo
,
SubcomponentStates
:
nil
,
// todo add subcomponents states
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
return
ret
,
nil
}
func
(
i
*
IndexNode
)
GetTimeTickChannel
()
(
string
,
error
)
{
return
""
,
nil
}
func
(
i
*
IndexNode
)
GetStatisticsChannel
()
(
string
,
error
)
{
return
""
,
nil
}
internal/indexservice/indexservice.go
浏览文件 @
e83ac41b
...
...
@@ -26,8 +26,11 @@ const (
type
IndexService
struct
{
nodeClients
*
PriorityQueue
nodeStates
map
[
UniqueID
]
*
internalpb2
.
ComponentStates
state
internalpb2
.
StateCode
ID
UniqueID
//factory method
loopCtx
context
.
Context
loopCancel
func
()
loopWg
sync
.
WaitGroup
...
...
@@ -85,6 +88,11 @@ func CreateIndexService(ctx context.Context) (*IndexService, error) {
return
nil
,
err
}
i
.
ID
,
err
=
i
.
idAllocator
.
AllocOne
()
if
err
!=
nil
{
return
nil
,
err
}
connectMinIOFn
:=
func
()
error
{
option
:=
&
miniokv
.
Option
{
Address
:
Params
.
MinIOAddress
,
...
...
@@ -139,7 +147,21 @@ func (i *IndexService) Stop() error {
}
func
(
i
*
IndexService
)
GetComponentStates
()
(
*
internalpb2
.
ComponentStates
,
error
)
{
panic
(
"implement me"
)
stateInfo
:=
&
internalpb2
.
ComponentInfo
{
NodeID
:
i
.
ID
,
Role
:
"IndexService"
,
StateCode
:
i
.
state
,
}
ret
:=
&
internalpb2
.
ComponentStates
{
State
:
stateInfo
,
SubcomponentStates
:
nil
,
// todo add subcomponents states
Status
:
&
commonpb
.
Status
{
ErrorCode
:
commonpb
.
ErrorCode_SUCCESS
,
},
}
return
ret
,
nil
}
func
(
i
*
IndexService
)
GetTimeTickChannel
()
(
string
,
error
)
{
...
...
@@ -147,7 +169,7 @@ func (i *IndexService) GetTimeTickChannel() (string, error) {
}
func
(
i
*
IndexService
)
GetStatisticsChannel
()
(
string
,
error
)
{
panic
(
"implement me"
)
return
""
,
nil
}
func
(
i
*
IndexService
)
BuildIndex
(
req
*
indexpb
.
BuildIndexRequest
)
(
*
indexpb
.
BuildIndexResponse
,
error
)
{
...
...
internal/proto/index_service.proto
浏览文件 @
e83ac41b
...
...
@@ -6,6 +6,7 @@ option go_package = "github.com/zilliztech/milvus-distributed/internal/proto/ind
import
"common.proto"
;
import
"internal.proto"
;
import
"milvus.proto"
;
message
RegisterNodeRequest
{
...
...
@@ -93,6 +94,10 @@ service IndexService {
rpc
GetIndexStates
(
IndexStatesRequest
)
returns
(
IndexStatesResponse
)
{}
rpc
GetIndexFilePaths
(
IndexFilePathsRequest
)
returns
(
IndexFilePathsResponse
){}
rpc
NotifyBuildIndex
(
BuildIndexNotification
)
returns
(
common.Status
)
{}
rpc
GetComponentStates
(
common.Empty
)
returns
(
internal.ComponentStates
)
{}
rpc
GetTimeTickChannel
(
common.Empty
)
returns
(
milvus.StringResponse
)
{}
rpc
GetStatisticsChannel
(
common.Empty
)
returns
(
milvus.StringResponse
){}
}
...
...
@@ -106,4 +111,7 @@ service IndexNode {
*/
rpc
BuildIndex
(
BuildIndexCmd
)
returns
(
common.Status
){}
rpc
GetComponentStates
(
common.Empty
)
returns
(
internal.ComponentStates
)
{}
rpc
GetTimeTickChannel
(
common.Empty
)
returns
(
milvus.StringResponse
)
{}
rpc
GetStatisticsChannel
(
common.Empty
)
returns
(
milvus.StringResponse
){}
}
internal/proto/indexpb/index_service.pb.go
浏览文件 @
e83ac41b
此差异已折叠。
点击以展开。
internal/util/typeutil/interface.go
浏览文件 @
e83ac41b
...
...
@@ -21,6 +21,7 @@ type Component interface {
type
IndexNodeInterface
interface
{
Service
Component
BuildIndex
(
req
*
indexpb
.
BuildIndexCmd
)
(
*
commonpb
.
Status
,
error
)
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录