Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
ae26a815
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ae26a815
编写于
11月 26, 2020
作者:
N
neza2017
提交者:
yefu.chen
11月 26, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Change to use Marshal as text
Signed-off-by:
N
neza2017
<
yefu.chen@zilliz.com
>
上级
f4c643f1
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
19 addition
and
144 deletion
+19
-144
configs/advanced/master.yaml
configs/advanced/master.yaml
+1
-4
configs/advanced/proxy.yaml
configs/advanced/proxy.yaml
+0
-1
internal/master/meta_table.go
internal/master/meta_table.go
+12
-41
internal/master/meta_table_test.go
internal/master/meta_table_test.go
+0
-41
internal/master/param_table.go
internal/master/param_table.go
+0
-26
internal/master/partition_task.go
internal/master/partition_task.go
+3
-5
internal/master/partition_task_test.go
internal/master/partition_task_test.go
+2
-5
internal/master/segment_manager_test.go
internal/master/segment_manager_test.go
+0
-3
internal/proxy/paramtable.go
internal/proxy/paramtable.go
+0
-12
internal/proxy/task.go
internal/proxy/task.go
+0
-5
internal/proxy/validate_util.go
internal/proxy/validate_util.go
+1
-1
未找到文件。
configs/advanced/master.yaml
浏览文件 @
ae26a815
...
...
@@ -20,7 +20,4 @@ master:
minIDAssignCnt
:
1024
maxIDAssignCnt
:
16384
# old name: segmentExpireDuration: 2000
IDAssignExpiration
:
2000
# ms
maxPartitionNum
:
4096
defaultPartitionTag
:
_default
\ No newline at end of file
IDAssignExpiration
:
2000
# ms
\ No newline at end of file
configs/advanced/proxy.yaml
浏览文件 @
ae26a815
...
...
@@ -28,4 +28,3 @@ proxy:
bufSize
:
512
maxNameLength
:
255
maxFieldNum
:
64
\ No newline at end of file
internal/master/meta_table.go
浏览文件 @
ae26a815
...
...
@@ -52,7 +52,7 @@ func (mt *metaTable) reloadFromKV() error {
for
_
,
value
:=
range
values
{
tenantMeta
:=
pb
.
TenantMeta
{}
err
:=
proto
.
Unmarshal
([]
byte
(
value
)
,
&
tenantMeta
)
err
:=
proto
.
Unmarshal
Text
(
value
,
&
tenantMeta
)
if
err
!=
nil
{
return
err
}
...
...
@@ -66,7 +66,7 @@ func (mt *metaTable) reloadFromKV() error {
for
_
,
value
:=
range
values
{
proxyMeta
:=
pb
.
ProxyMeta
{}
err
=
proto
.
Unmarshal
([]
byte
(
value
)
,
&
proxyMeta
)
err
=
proto
.
Unmarshal
Text
(
value
,
&
proxyMeta
)
if
err
!=
nil
{
return
err
}
...
...
@@ -80,7 +80,7 @@ func (mt *metaTable) reloadFromKV() error {
for
_
,
value
:=
range
values
{
collectionMeta
:=
pb
.
CollectionMeta
{}
err
=
proto
.
Unmarshal
([]
byte
(
value
)
,
&
collectionMeta
)
err
=
proto
.
Unmarshal
Text
(
value
,
&
collectionMeta
)
if
err
!=
nil
{
return
err
}
...
...
@@ -95,7 +95,7 @@ func (mt *metaTable) reloadFromKV() error {
for
_
,
value
:=
range
values
{
segmentMeta
:=
pb
.
SegmentMeta
{}
err
=
proto
.
Unmarshal
([]
byte
(
value
)
,
&
segmentMeta
)
err
=
proto
.
Unmarshal
Text
(
value
,
&
segmentMeta
)
if
err
!=
nil
{
return
err
}
...
...
@@ -107,10 +107,7 @@ func (mt *metaTable) reloadFromKV() error {
// metaTable.ddLock.Lock() before call this function
func
(
mt
*
metaTable
)
saveCollectionMeta
(
coll
*
pb
.
CollectionMeta
)
error
{
collBytes
,
err
:=
proto
.
Marshal
(
coll
)
if
err
!=
nil
{
return
err
}
collBytes
:=
proto
.
MarshalTextString
(
coll
)
mt
.
collID2Meta
[
coll
.
ID
]
=
*
coll
mt
.
collName2ID
[
coll
.
Schema
.
Name
]
=
coll
.
ID
return
mt
.
client
.
Save
(
"/collection/"
+
strconv
.
FormatInt
(
coll
.
ID
,
10
),
string
(
collBytes
))
...
...
@@ -118,10 +115,7 @@ func (mt *metaTable) saveCollectionMeta(coll *pb.CollectionMeta) error {
// metaTable.ddLock.Lock() before call this function
func
(
mt
*
metaTable
)
saveSegmentMeta
(
seg
*
pb
.
SegmentMeta
)
error
{
segBytes
,
err
:=
proto
.
Marshal
(
seg
)
if
err
!=
nil
{
return
err
}
segBytes
:=
proto
.
MarshalTextString
(
seg
)
mt
.
segID2Meta
[
seg
.
SegmentID
]
=
*
seg
...
...
@@ -136,10 +130,7 @@ func (mt *metaTable) saveCollectionAndDeleteSegmentsMeta(coll *pb.CollectionMeta
}
kvs
:=
make
(
map
[
string
]
string
)
collStrs
,
err
:=
proto
.
Marshal
(
coll
)
if
err
!=
nil
{
return
err
}
collStrs
:=
proto
.
MarshalTextString
(
coll
)
kvs
[
"/collection/"
+
strconv
.
FormatInt
(
coll
.
ID
,
10
)]
=
string
(
collStrs
)
...
...
@@ -159,19 +150,15 @@ func (mt *metaTable) saveCollectionAndDeleteSegmentsMeta(coll *pb.CollectionMeta
// metaTable.ddLock.Lock() before call this function
func
(
mt
*
metaTable
)
saveCollectionsAndSegmentsMeta
(
coll
*
pb
.
CollectionMeta
,
seg
*
pb
.
SegmentMeta
)
error
{
kvs
:=
make
(
map
[
string
]
string
)
collBytes
,
err
:=
proto
.
Marshal
(
coll
)
if
err
!=
nil
{
return
err
}
collBytes
:=
proto
.
MarshalTextString
(
coll
)
kvs
[
"/collection/"
+
strconv
.
FormatInt
(
coll
.
ID
,
10
)]
=
string
(
collBytes
)
mt
.
collID2Meta
[
coll
.
ID
]
=
*
coll
mt
.
collName2ID
[
coll
.
Schema
.
Name
]
=
coll
.
ID
segBytes
,
err
:=
proto
.
Marshal
(
seg
)
if
err
!=
nil
{
return
err
}
segBytes
:=
proto
.
MarshalTextString
(
seg
)
kvs
[
"/segment/"
+
strconv
.
FormatInt
(
seg
.
SegmentID
,
10
)]
=
string
(
segBytes
)
mt
.
segID2Meta
[
seg
.
SegmentID
]
=
*
seg
...
...
@@ -220,7 +207,7 @@ func (mt *metaTable) AddCollection(coll *pb.CollectionMeta) error {
}
if
len
(
coll
.
PartitionTags
)
==
0
{
coll
.
PartitionTags
=
append
(
coll
.
PartitionTags
,
Params
.
DefaultPartitionTag
)
coll
.
PartitionTags
=
append
(
coll
.
PartitionTags
,
"default"
)
}
_
,
ok
:=
mt
.
collName2ID
[
coll
.
Schema
.
Name
]
if
ok
{
...
...
@@ -292,10 +279,6 @@ func (mt *metaTable) AddPartition(collID UniqueID, tag string) error {
return
errors
.
Errorf
(
"can't find collection. id = "
+
strconv
.
FormatInt
(
collID
,
10
))
}
// number of partition tags (except _default) should be limited to 4096 by default
if
int64
(
len
(
coll
.
PartitionTags
))
>
Params
.
MaxPartitionNum
{
return
errors
.
New
(
"maximum partition's number should be limit to "
+
strconv
.
FormatInt
(
Params
.
MaxPartitionNum
,
10
))
}
for
_
,
t
:=
range
coll
.
PartitionTags
{
if
t
==
tag
{
return
errors
.
Errorf
(
"partition already exists."
)
...
...
@@ -330,29 +313,17 @@ func (mt *metaTable) DeletePartition(collID UniqueID, tag string) error {
mt
.
ddLock
.
Lock
()
defer
mt
.
ddLock
.
Unlock
()
if
tag
==
Params
.
DefaultPartitionTag
{
return
errors
.
New
(
"default partition cannot be deleted"
)
}
collMeta
,
ok
:=
mt
.
collID2Meta
[
collID
]
if
!
ok
{
return
errors
.
Errorf
(
"can't find collection. id = "
+
strconv
.
FormatInt
(
collID
,
10
))
}
// check tag exists
exist
:=
false
pt
:=
make
([]
string
,
0
,
len
(
collMeta
.
PartitionTags
))
for
_
,
t
:=
range
collMeta
.
PartitionTags
{
if
t
!=
tag
{
pt
=
append
(
pt
,
t
)
}
else
{
exist
=
true
}
}
if
!
exist
{
return
errors
.
New
(
"partition "
+
tag
+
" does not exist"
)
}
if
len
(
pt
)
==
len
(
collMeta
.
PartitionTags
)
{
return
nil
}
...
...
internal/master/meta_table_test.go
浏览文件 @
ae26a815
...
...
@@ -3,7 +3,6 @@ package master
import
(
"context"
"reflect"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
...
...
@@ -239,10 +238,6 @@ func TestMetaTable_DeletePartition(t *testing.T) {
assert
.
Equal
(
t
,
1
,
len
(
meta
.
collName2ID
))
assert
.
Equal
(
t
,
1
,
len
(
meta
.
collID2Meta
))
assert
.
Equal
(
t
,
1
,
len
(
meta
.
segID2Meta
))
// delete not exist
err
=
meta
.
DeletePartition
(
100
,
"not_exist"
)
assert
.
NotNil
(
t
,
err
)
}
func
TestMetaTable_Segment
(
t
*
testing
.
T
)
{
...
...
@@ -371,39 +366,3 @@ func TestMetaTable_UpdateSegment(t *testing.T) {
assert
.
Nil
(
t
,
err
)
assert
.
Equal
(
t
,
seg
.
NumRows
,
int64
(
210
))
}
func
TestMetaTable_AddPartition_Limit
(
t
*
testing
.
T
)
{
Init
()
Params
.
MaxPartitionNum
=
256
// adding 4096 partitions is too slow
etcdAddr
:=
Params
.
EtcdAddress
cli
,
err
:=
clientv3
.
New
(
clientv3
.
Config
{
Endpoints
:
[]
string
{
etcdAddr
}})
assert
.
Nil
(
t
,
err
)
etcdKV
:=
kv
.
NewEtcdKV
(
cli
,
"/etcd/test/root"
)
_
,
err
=
cli
.
Delete
(
context
.
TODO
(),
"/etcd/test/root"
,
clientv3
.
WithPrefix
())
assert
.
Nil
(
t
,
err
)
meta
,
err
:=
NewMetaTable
(
etcdKV
)
assert
.
Nil
(
t
,
err
)
defer
meta
.
client
.
Close
()
colMeta
:=
pb
.
CollectionMeta
{
ID
:
100
,
Schema
:
&
schemapb
.
CollectionSchema
{
Name
:
"coll1"
,
},
CreateTime
:
0
,
SegmentIDs
:
[]
UniqueID
{},
PartitionTags
:
[]
string
{},
}
err
=
meta
.
AddCollection
(
&
colMeta
)
assert
.
Nil
(
t
,
err
)
for
i
:=
0
;
i
<
int
(
Params
.
MaxPartitionNum
);
i
++
{
err
:=
meta
.
AddPartition
(
100
,
"partition_"
+
strconv
.
Itoa
(
i
))
assert
.
Nil
(
t
,
err
)
}
err
=
meta
.
AddPartition
(
100
,
"partition_limit"
)
assert
.
NotNil
(
t
,
err
)
}
internal/master/param_table.go
浏览文件 @
ae26a815
...
...
@@ -43,9 +43,6 @@ type ParamTable struct {
K2SChannelNames
[]
string
QueryNodeStatsChannelName
string
MsgChannelSubName
string
MaxPartitionNum
int64
DefaultPartitionTag
string
}
var
Params
ParamTable
...
...
@@ -94,8 +91,6 @@ func (p *ParamTable) Init() {
p
.
initK2SChannelNames
()
p
.
initQueryNodeStatsChannelName
()
p
.
initMsgChannelSubName
()
p
.
initMaxPartitionNum
()
p
.
initDefaultPartitionTag
()
}
func
(
p
*
ParamTable
)
initAddress
()
{
...
...
@@ -416,24 +411,3 @@ func (p *ParamTable) initK2SChannelNames() {
}
p
.
K2SChannelNames
=
channels
}
func
(
p
*
ParamTable
)
initMaxPartitionNum
()
{
str
,
err
:=
p
.
Load
(
"master.maxPartitionNum"
)
if
err
!=
nil
{
panic
(
err
)
}
maxPartitionNum
,
err
:=
strconv
.
ParseInt
(
str
,
10
,
64
)
if
err
!=
nil
{
panic
(
err
)
}
p
.
MaxPartitionNum
=
maxPartitionNum
}
func
(
p
*
ParamTable
)
initDefaultPartitionTag
()
{
defaultTag
,
err
:=
p
.
Load
(
"master.defaultPartitionTag"
)
if
err
!=
nil
{
panic
(
err
)
}
p
.
DefaultPartitionTag
=
defaultTag
}
internal/master/partition_task.go
浏览文件 @
ae26a815
...
...
@@ -191,12 +191,10 @@ func (t *showPartitionTask) Execute() error {
return
errors
.
New
(
"null request"
)
}
collMeta
,
err
:=
t
.
mt
.
GetCollectionByName
(
t
.
req
.
CollectionName
.
CollectionName
)
if
err
!=
nil
{
return
err
}
partitions
:=
make
([]
string
,
0
)
partitions
=
append
(
partitions
,
collMeta
.
PartitionTags
...
)
for
_
,
collection
:=
range
t
.
mt
.
collID2Meta
{
partitions
=
append
(
partitions
,
collection
.
PartitionTags
...
)
}
stringListResponse
:=
servicepb
.
StringListResponse
{
Status
:
&
commonpb
.
Status
{
...
...
internal/master/partition_task_test.go
浏览文件 @
ae26a815
...
...
@@ -60,9 +60,6 @@ func TestMaster_Partition(t *testing.T) {
K2SChannelNames
:
[]
string
{
"k2s0"
,
"k2s1"
},
QueryNodeStatsChannelName
:
"statistic"
,
MsgChannelSubName
:
Params
.
MsgChannelSubName
,
MaxPartitionNum
:
int64
(
4096
),
DefaultPartitionTag
:
"_default"
,
}
port
:=
10000
+
rand
.
Intn
(
1000
)
...
...
@@ -215,7 +212,7 @@ func TestMaster_Partition(t *testing.T) {
//assert.Equal(t, collMeta.PartitionTags[0], "partition1")
//assert.Equal(t, collMeta.PartitionTags[1], "partition2")
assert
.
ElementsMatch
(
t
,
[]
string
{
"
_
default"
,
"partition1"
,
"partition2"
},
collMeta
.
PartitionTags
)
assert
.
ElementsMatch
(
t
,
[]
string
{
"default"
,
"partition1"
,
"partition2"
},
collMeta
.
PartitionTags
)
showPartitionReq
:=
internalpb
.
ShowPartitionRequest
{
MsgType
:
internalpb
.
MsgType_kShowPartitions
,
...
...
@@ -227,7 +224,7 @@ func TestMaster_Partition(t *testing.T) {
stringList
,
err
:=
cli
.
ShowPartitions
(
ctx
,
&
showPartitionReq
)
assert
.
Nil
(
t
,
err
)
assert
.
ElementsMatch
(
t
,
[]
string
{
"
_
default"
,
"partition1"
,
"partition2"
},
stringList
.
Values
)
assert
.
ElementsMatch
(
t
,
[]
string
{
"default"
,
"partition1"
,
"partition2"
},
stringList
.
Values
)
showPartitionReq
=
internalpb
.
ShowPartitionRequest
{
MsgType
:
internalpb
.
MsgType_kShowPartitions
,
...
...
internal/master/segment_manager_test.go
浏览文件 @
ae26a815
...
...
@@ -261,9 +261,6 @@ func startupMaster() {
K2SChannelNames
:
[]
string
{
"k2s0"
,
"k2s1"
},
QueryNodeStatsChannelName
:
"statistic"
,
MsgChannelSubName
:
Params
.
MsgChannelSubName
,
MaxPartitionNum
:
int64
(
4096
),
DefaultPartitionTag
:
"_default"
,
}
master
,
err
=
CreateServer
(
ctx
)
...
...
internal/proxy/paramtable.go
浏览文件 @
ae26a815
...
...
@@ -463,15 +463,3 @@ func (pt *ParamTable) MaxNameLength() int64 {
}
return
maxNameLength
}
func
(
pt
*
ParamTable
)
MaxFieldNum
()
int64
{
str
,
err
:=
pt
.
Load
(
"proxy.maxFieldNum"
)
if
err
!=
nil
{
panic
(
err
)
}
maxFieldNum
,
err
:=
strconv
.
ParseInt
(
str
,
10
,
64
)
if
err
!=
nil
{
panic
(
err
)
}
return
maxFieldNum
}
internal/proxy/task.go
浏览文件 @
ae26a815
...
...
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"log"
"strconv"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/allocator"
...
...
@@ -165,10 +164,6 @@ func (cct *CreateCollectionTask) SetTs(ts Timestamp) {
}
func
(
cct
*
CreateCollectionTask
)
PreExecute
()
error
{
if
int64
(
len
(
cct
.
schema
.
Fields
))
>
Params
.
MaxFieldNum
()
{
return
errors
.
New
(
"maximum field's number should be limited to "
+
strconv
.
FormatInt
(
Params
.
MaxFieldNum
(),
10
))
}
// validate collection name
if
err
:=
ValidateCollectionName
(
cct
.
schema
.
Name
);
err
!=
nil
{
return
err
...
...
internal/proxy/validate_util.go
浏览文件 @
ae26a815
...
...
@@ -68,7 +68,7 @@ func ValidatePartitionTag(partitionTag string, strictCheck bool) error {
if
strictCheck
{
firstChar
:=
partitionTag
[
0
]
if
firstChar
!=
'_'
&&
!
isAlpha
(
firstChar
)
&&
!
isNumber
(
firstChar
)
{
if
firstChar
!=
'_'
&&
!
isAlpha
(
firstChar
)
{
msg
:=
invalidMsg
+
"The first character of a partition tag must be an underscore or letter."
return
errors
.
New
(
msg
)
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录