Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7744573d
M
milvus
项目概览
milvus
/
milvus
9 个月 前同步成功
通知
260
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
7744573d
编写于
5月 18, 2023
作者:
M
MrPresent-Han
提交者:
GitHub
5月 18, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
support parms for import maxfilesize(#24191) (#24192)
Signed-off-by:
N
MrPresent-Han
<
chun.han@zilliz.com
>
上级
b1afd3ea
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
22 addition
and
12 deletion
+22
-12
configs/milvus.yaml
configs/milvus.yaml
+3
-0
internal/util/importutil/import_wrapper.go
internal/util/importutil/import_wrapper.go
+4
-8
internal/util/importutil/import_wrapper_test.go
internal/util/importutil/import_wrapper_test.go
+4
-2
internal/util/importutil/numpy_parser.go
internal/util/importutil/numpy_parser.go
+2
-2
pkg/util/paramtable/component_param.go
pkg/util/paramtable/component_param.go
+9
-0
未找到文件。
configs/milvus.yaml
浏览文件 @
7744573d
...
...
@@ -420,6 +420,9 @@ common:
# need to set a separated topic to stand for currently consumed timestamp for each channel
timeticker
:
"
timetick-channel"
ImportMaxFileSize
:
17179869184
# 16 * 1024 * 1024 * 1024
# max file size to import for bulkInsert
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:
# 1. TT protection;
...
...
internal/util/importutil/import_wrapper.go
浏览文件 @
7744573d
...
...
@@ -29,6 +29,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/internal/querycoordv2/params"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
...
...
@@ -43,11 +44,6 @@ const (
// supposed size of a single block, to control a binlog file size, the max biglog file size is no more than 2*SingleBlockSize
SingleBlockSize
=
16
*
1024
*
1024
// 16MB
// this limitation is to avoid this OOM risk:
// for column-based file, we read all its data into memory, if user input a large file, the read() method may
// cost extra memory and lear to OOM.
MaxFileSize
=
16
*
1024
*
1024
*
1024
// 16GB
// this limitation is to avoid this OOM risk:
// simetimes system segment max size is a large number, a single segment fields data might cause OOM.
// flush the segment when its data reach this limitation, let the compaction to compact it later.
...
...
@@ -239,10 +235,10 @@ func (p *ImportWrapper) fileValidation(filePaths []string) (bool, error) {
return
rowBased
,
fmt
.
Errorf
(
"the file '%s' size is zero"
,
filePath
)
}
if
size
>
MaxFileSize
{
if
size
>
params
.
Params
.
CommonCfg
.
ImportMaxFileSize
.
GetAsInt64
()
{
log
.
Error
(
"import wrapper: file size exceeds the maximum size"
,
zap
.
String
(
"filePath"
,
filePath
),
zap
.
Int64
(
"fileSize"
,
size
),
zap
.
Int64
(
"MaxFileSize"
,
MaxFileSize
))
return
rowBased
,
fmt
.
Errorf
(
"the file '%s' size exceeds the maximum size: %d bytes"
,
filePath
,
MaxFileSize
)
zap
.
Int64
(
"fileSize"
,
size
),
zap
.
Int64
(
"MaxFileSize"
,
params
.
Params
.
CommonCfg
.
ImportMaxFileSize
.
GetAsInt64
()
))
return
rowBased
,
fmt
.
Errorf
(
"the file '%s' size exceeds the maximum size: %d bytes"
,
filePath
,
params
.
Params
.
CommonCfg
.
ImportMaxFileSize
.
GetAsInt64
()
)
}
totalSize
+=
size
}
...
...
internal/util/importutil/import_wrapper_test.go
浏览文件 @
7744573d
...
...
@@ -37,6 +37,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/internal/querycoordv2/params"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/timerecord"
...
...
@@ -228,6 +229,7 @@ func Test_ImportWrapperRowBased(t *testing.T) {
err
:=
os
.
MkdirAll
(
TempFilesPath
,
os
.
ModePerm
)
assert
.
Nil
(
t
,
err
)
defer
os
.
RemoveAll
(
TempFilesPath
)
params
.
Params
.
Init
()
// NewDefaultFactory() use "/tmp/milvus" as default root path, and cannot specify root path
// NewChunkManagerFactory() can specify the root path
...
...
@@ -618,7 +620,7 @@ func Test_ImportWrapperFileValidation(t *testing.T) {
segmentSize
:=
512
// unit: MB
wrapper
:=
NewImportWrapper
(
ctx
,
schema
,
int32
(
shardNum
),
int64
(
segmentSize
),
idAllocator
,
cm
,
nil
,
nil
)
params
.
Params
.
Init
()
t
.
Run
(
"unsupported file type"
,
func
(
t
*
testing
.
T
)
{
files
:=
[]
string
{
"uid.txt"
}
rowBased
,
err
:=
wrapper
.
fileValidation
(
files
)
...
...
@@ -675,7 +677,7 @@ func Test_ImportWrapperFileValidation(t *testing.T) {
t
.
Run
(
"file size exceed MaxFileSize limit"
,
func
(
t
*
testing
.
T
)
{
files
:=
[]
string
{
"a/1.json"
}
cm
.
size
=
MaxFileSize
+
1
cm
.
size
=
params
.
Params
.
CommonCfg
.
ImportMaxFileSize
.
GetAsInt64
()
+
1
wrapper
=
NewImportWrapper
(
ctx
,
schema
,
int32
(
shardNum
),
int64
(
segmentSize
),
idAllocator
,
cm
,
nil
,
nil
)
rowBased
,
err
:=
wrapper
.
fileValidation
(
files
)
assert
.
NotNil
(
t
,
err
)
...
...
internal/util/importutil/numpy_parser.go
浏览文件 @
7744573d
...
...
@@ -117,7 +117,7 @@ func (p *NumpyParser) IDRange() []int64 {
// Parse is the function entry
func
(
p
*
NumpyParser
)
Parse
(
filePaths
[]
string
)
error
{
// check redundant files for column-based import
// if the field is primary key and auto
id
is false, the file is required
// if the field is primary key and auto
ID
is false, the file is required
// any redundant file is not allowed
err
:=
p
.
validateFileNames
(
filePaths
)
if
err
!=
nil
{
...
...
@@ -126,7 +126,7 @@ func (p *NumpyParser) Parse(filePaths []string) error {
// open files and verify file header
readers
,
err
:=
p
.
createReaders
(
filePaths
)
// make sure all the files are closed fin
i
ally, must call this method before the function return
// make sure all the files are closed finally, must call this method before the function return
defer
closeReaders
(
readers
)
if
err
!=
nil
{
return
err
...
...
pkg/util/paramtable/component_param.go
浏览文件 @
7744573d
...
...
@@ -216,6 +216,8 @@ type commonConfig struct {
TimeTicker
ParamItem
`refreshable:"true"`
JSONMaxLength
ParamItem
`refreshable:"false"`
ImportMaxFileSize
ParamItem
`refreshable:"true"`
}
func
(
p
*
commonConfig
)
init
(
base
*
BaseTable
)
{
...
...
@@ -643,6 +645,13 @@ like the old password verification when updating the credential`,
DefaultValue
:
fmt
.
Sprint
(
64
<<
10
),
}
p
.
JSONMaxLength
.
Init
(
base
.
mgr
)
p
.
ImportMaxFileSize
=
ParamItem
{
Key
:
"common.ImportMaxFileSize"
,
Version
:
"2.2.9"
,
DefaultValue
:
fmt
.
Sprint
(
16
<<
30
),
}
p
.
ImportMaxFileSize
.
Init
(
base
.
mgr
)
}
type
traceConfig
struct
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录