diff --git a/internal/proxy/validate_util.go b/internal/proxy/validate_util.go index 7096682872fee384c2030880295af6473a143333..6ba3995257f0cea237c2749e34b801759d888591 100644 --- a/internal/proxy/validate_util.go +++ b/internal/proxy/validate_util.go @@ -6,6 +6,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/schemapb" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/merr" + "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -63,8 +64,11 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col if err := v.checkVarCharFieldData(field, fieldSchema); err != nil { return err } + case schemapb.DataType_JSON: + if err := v.checkJSONFieldData(field, fieldSchema); err != nil { + return err + } default: - } } @@ -172,7 +176,21 @@ func (v *validateUtil) checkVarCharFieldData(field *schemapb.FieldData, fieldSch return nil } -func verifyLengthPerRow(strArr []string, maxLength int64) error { +func (v *validateUtil) checkJSONFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error { + jsonArray := field.GetScalars().GetJsonData().GetData() + if jsonArray == nil { + msg := fmt.Sprintf("varchar field '%v' is illegal, array type mismatch", field.GetFieldName()) + return merr.WrapErrParameterInvalid("need string array", "got nil", msg) + } + + if v.checkMaxLen { + return verifyLengthPerRow(jsonArray, paramtable.Get().CommonCfg.JSONMaxLength.GetAsInt64()) + } + + return nil +} + +func verifyLengthPerRow[E interface{ ~string | ~[]byte }](strArr []E, maxLength int64) error { for i, s := range strArr { if int64(len(s)) > maxLength { msg := fmt.Sprintf("the length (%d) of %dth string exceeds max length (%d)", len(s), i, maxLength) diff --git a/internal/proxy/validate_util_test.go b/internal/proxy/validate_util_test.go index be1a54d92c67a867528df1b251b51852b5ea22d9..6dc666caa9c88d8488d176afb8bc50926c0f7c4f 100644 --- a/internal/proxy/validate_util_test.go +++ b/internal/proxy/validate_util_test.go @@ -8,13 +8,14 @@ import ( "github.com/milvus-io/milvus-proto/go-api/commonpb" "github.com/milvus-io/milvus-proto/go-api/schemapb" + "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) func Test_verifyLengthPerRow(t *testing.T) { maxLength := 16 - assert.NoError(t, verifyLengthPerRow(nil, int64(maxLength))) + assert.NoError(t, verifyLengthPerRow[string](nil, int64(maxLength))) assert.NoError(t, verifyLengthPerRow([]string{"111111", "22222"}, int64(maxLength))) @@ -596,6 +597,8 @@ func Test_validateUtil_checkAligned(t *testing.T) { } func Test_validateUtil_Validate(t *testing.T) { + paramtable.Init() + t.Run("nil schema", func(t *testing.T) { data := []*schemapb.FieldData{ { @@ -816,9 +819,36 @@ func Test_validateUtil_Validate(t *testing.T) { } v := newValidateUtil(withNANCheck(), withMaxLenCheck()) - err := v.Validate(data, schema, 2) + assert.Error(t, err) + // Validate JSON length + longBytes := make([]byte, paramtable.Get().CommonCfg.JSONMaxLength.GetAsInt()+1) + data = []*schemapb.FieldData{ + { + FieldName: "json", + Type: schemapb.DataType_JSON, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_JsonData{ + JsonData: &schemapb.JSONArray{ + Data: [][]byte{longBytes, longBytes}, + }, + }, + }, + }, + }, + } + schema = &schemapb.CollectionSchema{ + Fields: []*schemapb.FieldSchema{ + { + Name: "json", + FieldID: 104, + DataType: schemapb.DataType_JSON, + }, + }, + } + err = v.Validate(data, schema, 2) assert.Error(t, err) }) @@ -831,7 +861,7 @@ func Test_validateUtil_Validate(t *testing.T) { Vectors: &schemapb.VectorField{ Data: &schemapb.VectorField_FloatVector{ FloatVector: &schemapb.FloatArray{ - Data: generateFloatVectors(10, 8), + Data: generateFloatVectors(2, 8), }, }, }, @@ -843,7 +873,7 @@ func Test_validateUtil_Validate(t *testing.T) { Field: &schemapb.FieldData_Vectors{ Vectors: &schemapb.VectorField{ Data: &schemapb.VectorField_BinaryVector{ - BinaryVector: generateBinaryVectors(10, 8), + BinaryVector: generateBinaryVectors(2, 8), }, }, }, @@ -855,7 +885,20 @@ func Test_validateUtil_Validate(t *testing.T) { Scalars: &schemapb.ScalarField{ Data: &schemapb.ScalarField_StringData{ StringData: &schemapb.StringArray{ - Data: generateVarCharArray(10, 8), + Data: generateVarCharArray(2, 8), + }, + }, + }, + }, + }, + { + FieldName: "test4", + Type: schemapb.DataType_JSON, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_JsonData{ + JsonData: &schemapb.JSONArray{ + Data: [][]byte{[]byte("{}"), []byte("{}")}, }, }, }, @@ -898,12 +941,17 @@ func Test_validateUtil_Validate(t *testing.T) { }, }, }, + { + Name: "test4", + FieldID: 104, + DataType: schemapb.DataType_JSON, + }, }, } v := newValidateUtil(withNANCheck(), withMaxLenCheck()) - err := v.Validate(data, schema, 10) + err := v.Validate(data, schema, 2) assert.NoError(t, err) }) diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 690e36de13956f18cc34ef48da4a6e0f9e93e3f9..07833d375de3127df43b5f90a9753f8679af922a 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -214,6 +214,8 @@ type commonConfig struct { PreCreatedTopicEnabled ParamItem `refreshable:"true"` TopicNames ParamItem `refreshable:"true"` TimeTicker ParamItem `refreshable:"true"` + + JSONMaxLength ParamItem `refreshable:"false"` } func (p *commonConfig) init(base *BaseTable) { @@ -634,6 +636,13 @@ like the old password verification when updating the credential`, Version: "2.3.0", } p.TimeTicker.Init(base.mgr) + + p.JSONMaxLength = ParamItem{ + Key: "common.JSONMaxLength", + Version: "2.2.9", + DefaultValue: fmt.Sprint(64 << 10), + } + p.JSONMaxLength.Init(base.mgr) } type traceConfig struct {