Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
7d0c47dd
M
milvus
项目概览
milvus
/
milvus
11 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
7d0c47dd
编写于
5月 12, 2023
作者:
Y
yah01
提交者:
GitHub
5月 12, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Check the length of JSON (#24035)
Signed-off-by:
N
yah01
<
yang.cen@zilliz.com
>
上级
7e645dab
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
83 addition
and
8 deletion
+83
-8
internal/proxy/validate_util.go
internal/proxy/validate_util.go
+20
-2
internal/proxy/validate_util_test.go
internal/proxy/validate_util_test.go
+54
-6
pkg/util/paramtable/component_param.go
pkg/util/paramtable/component_param.go
+9
-0
未找到文件。
internal/proxy/validate_util.go
浏览文件 @
7d0c47dd
...
@@ -6,6 +6,7 @@ import (
...
@@ -6,6 +6,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
)
...
@@ -63,8 +64,11 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col
...
@@ -63,8 +64,11 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col
if
err
:=
v
.
checkVarCharFieldData
(
field
,
fieldSchema
);
err
!=
nil
{
if
err
:=
v
.
checkVarCharFieldData
(
field
,
fieldSchema
);
err
!=
nil
{
return
err
return
err
}
}
case
schemapb
.
DataType_JSON
:
if
err
:=
v
.
checkJSONFieldData
(
field
,
fieldSchema
);
err
!=
nil
{
return
err
}
default
:
default
:
}
}
}
}
...
@@ -172,7 +176,21 @@ func (v *validateUtil) checkVarCharFieldData(field *schemapb.FieldData, fieldSch
...
@@ -172,7 +176,21 @@ func (v *validateUtil) checkVarCharFieldData(field *schemapb.FieldData, fieldSch
return
nil
return
nil
}
}
func
verifyLengthPerRow
(
strArr
[]
string
,
maxLength
int64
)
error
{
func
(
v
*
validateUtil
)
checkJSONFieldData
(
field
*
schemapb
.
FieldData
,
fieldSchema
*
schemapb
.
FieldSchema
)
error
{
jsonArray
:=
field
.
GetScalars
()
.
GetJsonData
()
.
GetData
()
if
jsonArray
==
nil
{
msg
:=
fmt
.
Sprintf
(
"varchar field '%v' is illegal, array type mismatch"
,
field
.
GetFieldName
())
return
merr
.
WrapErrParameterInvalid
(
"need string array"
,
"got nil"
,
msg
)
}
if
v
.
checkMaxLen
{
return
verifyLengthPerRow
(
jsonArray
,
paramtable
.
Get
()
.
CommonCfg
.
JSONMaxLength
.
GetAsInt64
())
}
return
nil
}
func
verifyLengthPerRow
[
E
interface
{
~
string
|
~
[]
byte
}](
strArr
[]
E
,
maxLength
int64
)
error
{
for
i
,
s
:=
range
strArr
{
for
i
,
s
:=
range
strArr
{
if
int64
(
len
(
s
))
>
maxLength
{
if
int64
(
len
(
s
))
>
maxLength
{
msg
:=
fmt
.
Sprintf
(
"the length (%d) of %dth string exceeds max length (%d)"
,
len
(
s
),
i
,
maxLength
)
msg
:=
fmt
.
Sprintf
(
"the length (%d) of %dth string exceeds max length (%d)"
,
len
(
s
),
i
,
maxLength
)
...
...
internal/proxy/validate_util_test.go
浏览文件 @
7d0c47dd
...
@@ -8,13 +8,14 @@ import (
...
@@ -8,13 +8,14 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
)
func
Test_verifyLengthPerRow
(
t
*
testing
.
T
)
{
func
Test_verifyLengthPerRow
(
t
*
testing
.
T
)
{
maxLength
:=
16
maxLength
:=
16
assert
.
NoError
(
t
,
verifyLengthPerRow
(
nil
,
int64
(
maxLength
)))
assert
.
NoError
(
t
,
verifyLengthPerRow
[
string
]
(
nil
,
int64
(
maxLength
)))
assert
.
NoError
(
t
,
verifyLengthPerRow
([]
string
{
"111111"
,
"22222"
},
int64
(
maxLength
)))
assert
.
NoError
(
t
,
verifyLengthPerRow
([]
string
{
"111111"
,
"22222"
},
int64
(
maxLength
)))
...
@@ -596,6 +597,8 @@ func Test_validateUtil_checkAligned(t *testing.T) {
...
@@ -596,6 +597,8 @@ func Test_validateUtil_checkAligned(t *testing.T) {
}
}
func
Test_validateUtil_Validate
(
t
*
testing
.
T
)
{
func
Test_validateUtil_Validate
(
t
*
testing
.
T
)
{
paramtable
.
Init
()
t
.
Run
(
"nil schema"
,
func
(
t
*
testing
.
T
)
{
t
.
Run
(
"nil schema"
,
func
(
t
*
testing
.
T
)
{
data
:=
[]
*
schemapb
.
FieldData
{
data
:=
[]
*
schemapb
.
FieldData
{
{
{
...
@@ -816,9 +819,36 @@ func Test_validateUtil_Validate(t *testing.T) {
...
@@ -816,9 +819,36 @@ func Test_validateUtil_Validate(t *testing.T) {
}
}
v
:=
newValidateUtil
(
withNANCheck
(),
withMaxLenCheck
())
v
:=
newValidateUtil
(
withNANCheck
(),
withMaxLenCheck
())
err
:=
v
.
Validate
(
data
,
schema
,
2
)
err
:=
v
.
Validate
(
data
,
schema
,
2
)
assert
.
Error
(
t
,
err
)
// Validate JSON length
longBytes
:=
make
([]
byte
,
paramtable
.
Get
()
.
CommonCfg
.
JSONMaxLength
.
GetAsInt
()
+
1
)
data
=
[]
*
schemapb
.
FieldData
{
{
FieldName
:
"json"
,
Type
:
schemapb
.
DataType_JSON
,
Field
:
&
schemapb
.
FieldData_Scalars
{
Scalars
:
&
schemapb
.
ScalarField
{
Data
:
&
schemapb
.
ScalarField_JsonData
{
JsonData
:
&
schemapb
.
JSONArray
{
Data
:
[][]
byte
{
longBytes
,
longBytes
},
},
},
},
},
},
}
schema
=
&
schemapb
.
CollectionSchema
{
Fields
:
[]
*
schemapb
.
FieldSchema
{
{
Name
:
"json"
,
FieldID
:
104
,
DataType
:
schemapb
.
DataType_JSON
,
},
},
}
err
=
v
.
Validate
(
data
,
schema
,
2
)
assert
.
Error
(
t
,
err
)
assert
.
Error
(
t
,
err
)
})
})
...
@@ -831,7 +861,7 @@ func Test_validateUtil_Validate(t *testing.T) {
...
@@ -831,7 +861,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Vectors
:
&
schemapb
.
VectorField
{
Vectors
:
&
schemapb
.
VectorField
{
Data
:
&
schemapb
.
VectorField_FloatVector
{
Data
:
&
schemapb
.
VectorField_FloatVector
{
FloatVector
:
&
schemapb
.
FloatArray
{
FloatVector
:
&
schemapb
.
FloatArray
{
Data
:
generateFloatVectors
(
10
,
8
),
Data
:
generateFloatVectors
(
2
,
8
),
},
},
},
},
},
},
...
@@ -843,7 +873,7 @@ func Test_validateUtil_Validate(t *testing.T) {
...
@@ -843,7 +873,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Field
:
&
schemapb
.
FieldData_Vectors
{
Field
:
&
schemapb
.
FieldData_Vectors
{
Vectors
:
&
schemapb
.
VectorField
{
Vectors
:
&
schemapb
.
VectorField
{
Data
:
&
schemapb
.
VectorField_BinaryVector
{
Data
:
&
schemapb
.
VectorField_BinaryVector
{
BinaryVector
:
generateBinaryVectors
(
10
,
8
),
BinaryVector
:
generateBinaryVectors
(
2
,
8
),
},
},
},
},
},
},
...
@@ -855,7 +885,20 @@ func Test_validateUtil_Validate(t *testing.T) {
...
@@ -855,7 +885,20 @@ func Test_validateUtil_Validate(t *testing.T) {
Scalars
:
&
schemapb
.
ScalarField
{
Scalars
:
&
schemapb
.
ScalarField
{
Data
:
&
schemapb
.
ScalarField_StringData
{
Data
:
&
schemapb
.
ScalarField_StringData
{
StringData
:
&
schemapb
.
StringArray
{
StringData
:
&
schemapb
.
StringArray
{
Data
:
generateVarCharArray
(
10
,
8
),
Data
:
generateVarCharArray
(
2
,
8
),
},
},
},
},
},
{
FieldName
:
"test4"
,
Type
:
schemapb
.
DataType_JSON
,
Field
:
&
schemapb
.
FieldData_Scalars
{
Scalars
:
&
schemapb
.
ScalarField
{
Data
:
&
schemapb
.
ScalarField_JsonData
{
JsonData
:
&
schemapb
.
JSONArray
{
Data
:
[][]
byte
{[]
byte
(
"{}"
),
[]
byte
(
"{}"
)},
},
},
},
},
},
},
...
@@ -898,12 +941,17 @@ func Test_validateUtil_Validate(t *testing.T) {
...
@@ -898,12 +941,17 @@ func Test_validateUtil_Validate(t *testing.T) {
},
},
},
},
},
},
{
Name
:
"test4"
,
FieldID
:
104
,
DataType
:
schemapb
.
DataType_JSON
,
},
},
},
}
}
v
:=
newValidateUtil
(
withNANCheck
(),
withMaxLenCheck
())
v
:=
newValidateUtil
(
withNANCheck
(),
withMaxLenCheck
())
err
:=
v
.
Validate
(
data
,
schema
,
10
)
err
:=
v
.
Validate
(
data
,
schema
,
2
)
assert
.
NoError
(
t
,
err
)
assert
.
NoError
(
t
,
err
)
})
})
...
...
pkg/util/paramtable/component_param.go
浏览文件 @
7d0c47dd
...
@@ -214,6 +214,8 @@ type commonConfig struct {
...
@@ -214,6 +214,8 @@ type commonConfig struct {
PreCreatedTopicEnabled
ParamItem
`refreshable:"true"`
PreCreatedTopicEnabled
ParamItem
`refreshable:"true"`
TopicNames
ParamItem
`refreshable:"true"`
TopicNames
ParamItem
`refreshable:"true"`
TimeTicker
ParamItem
`refreshable:"true"`
TimeTicker
ParamItem
`refreshable:"true"`
JSONMaxLength
ParamItem
`refreshable:"false"`
}
}
func
(
p
*
commonConfig
)
init
(
base
*
BaseTable
)
{
func
(
p
*
commonConfig
)
init
(
base
*
BaseTable
)
{
...
@@ -634,6 +636,13 @@ like the old password verification when updating the credential`,
...
@@ -634,6 +636,13 @@ like the old password verification when updating the credential`,
Version
:
"2.3.0"
,
Version
:
"2.3.0"
,
}
}
p
.
TimeTicker
.
Init
(
base
.
mgr
)
p
.
TimeTicker
.
Init
(
base
.
mgr
)
p
.
JSONMaxLength
=
ParamItem
{
Key
:
"common.JSONMaxLength"
,
Version
:
"2.2.9"
,
DefaultValue
:
fmt
.
Sprint
(
64
<<
10
),
}
p
.
JSONMaxLength
.
Init
(
base
.
mgr
)
}
}
type
traceConfig
struct
{
type
traceConfig
struct
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录