Json key must be enclosed in double quotes (#24113)

Signed-off-by: Ncai.zhang <cai.zhang@zilliz.com>

internal/parser/planparserv2/Plan.g4

@@ -85,7 +85,7 @@ FloatingConstant:
 Identifier: Nondigit (Nondigit | Digit)*;
 
 StringLiteral: EncodingPrefix? '"' SCharSequence? '"';
-JSONIdentifier: Identifier('[' ('"' SCharSequence '"' | '\'' SCharSequence '\'' | SCharSequence | IntegerConstant) ']')+;
+JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
 
 fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
 
@@ -95,7 +95,7 @@ fragment SChar: ~["\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
 fragment Nondigit: [a-zA-Z_];
 fragment Digit: [0-9];
 fragment BinaryConstant: '0' [bB] [0-1]+;
-fragment DecimalConstant: NonzeroDigit Digit*;
+fragment DecimalConstant: NonzeroDigit Digit* | '0';
 fragment OctalConstant: '0' OctalDigit*;
 fragment HexadecimalConstant: '0' [xX] HexadecimalDigitSequence;
 fragment NonzeroDigit: [1-9];

internal/parser/planparserv2/parser_visitor.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/antlr/antlr4/runtime/Go/antlr"
-
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
 	parser "github.com/milvus-io/milvus/internal/parser/planparserv2/generated"
 	"github.com/milvus-io/milvus/internal/proto/planpb"
@@ -561,7 +560,10 @@ func (v *ParserVisitor) getChildColumnInfo(identifier, child antlr.TerminalNode)
 // VisitRange translates expr to range plan.
 func (v *ParserVisitor) VisitRange(ctx *parser.RangeContext) interface{} {
 	columnInfo, err := v.getChildColumnInfo(ctx.Identifier(), ctx.JSONIdentifier())
-	if columnInfo == nil || err != nil {
+	if err != nil {
+		return err
+	}
+	if columnInfo == nil {
 		return fmt.Errorf("range operations are only supported on single fields now, got: %s", ctx.Expr(1).GetText())
 	}
 
@@ -931,10 +933,8 @@ func (v *ParserVisitor) VisitBitOr(ctx *parser.BitOrContext) interface{} {
 */
 // More tests refer to plan_parser_v2_test.go::Test_JSONExpr
 func (v *ParserVisitor) getColumnInfoFromJSONIdentifier(identifier string) (*planpb.ColumnInfo, error) {
-	ss := strings.Split(identifier, "[")
-	fieldName := ss[0]
-	length := len(ss)
-	nestedPath := make([]string, 0, length)
+	fieldName := strings.Split(identifier, "[")[0]
+	nestedPath := make([]string, 0)
 	jsonField, err := v.schema.GetFieldFromNameDefaultJSON(fieldName)
 	if err != nil {
 		return nil, err
@@ -942,8 +942,18 @@ func (v *ParserVisitor) getColumnInfoFromJSONIdentifier(identifier string) (*pla
 	if fieldName != jsonField.Name {
 		nestedPath = append(nestedPath, fieldName)
 	}
-	for i := 1; i < length; i++ {
-		path := strings.Trim(strings.Trim(ss[i], "[]"), "\"")
+	jsonKeyStr := identifier[len(fieldName):]
+	ss := strings.Split(jsonKeyStr, "][")
+	for i := 0; i < len(ss); i++ {
+		path := strings.Trim(ss[i], "[]")
+		if path == "" {
+			return nil, fmt.Errorf("invalid identifier: %s", identifier)
+		}
+		if strings.HasPrefix(path, "\"") && strings.HasSuffix(path, "\"") {
+			path = path[1 : len(path)-1]
+		} else if _, err := strconv.ParseInt(path, 10, 64); err != nil {
+			return nil, fmt.Errorf("json key must be enclosed in double quotes: \"%s\"", path)
+		}
 		nestedPath = append(nestedPath, path)
 	}
 

internal/parser/planparserv2/plan_parser_v2_test.go

@@ -4,15 +4,12 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/milvus-io/milvus/pkg/log"
-	"go.uber.org/zap"
-
-	"github.com/milvus-io/milvus/internal/proto/planpb"
-
-	"github.com/stretchr/testify/assert"
-
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
+	"github.com/milvus-io/milvus/internal/proto/planpb"
+	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
 )
 
 func newTestSchema() *schemapb.CollectionSchema {
@@ -25,6 +22,12 @@ func newTestSchema() *schemapb.CollectionSchema {
 		newField := &schemapb.FieldSchema{
 			FieldID: int64(100 + value), Name: name + "Field", IsPrimaryKey: false, Description: "", DataType: dataType,
 		}
+		if value == int32(schemapb.DataType_JSON) {
+			newField = &schemapb.FieldSchema{
+				FieldID: int64(100 + value), Name: name + "Field", IsPrimaryKey: false, Description: "", DataType: dataType,
+				//IsDynamic: true,
+			}
+		}
 		fields = append(fields, newField)
 	}
 
@@ -687,6 +690,24 @@ func Test_JSONExpr(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
+	expr = `C["0"] not in [90, 91, 95, 97]`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
+	expr = `C[0] in [90, 91, 95, 97]`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
 	expr = `C["0"] > 90`
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
@@ -795,25 +816,16 @@ func Test_JSONExpr(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	expr = `exists JSONField["A"] > 10 `
-	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
-		Topk:         0,
-		MetricType:   "",
-		SearchParams: "",
-		RoundDecimal: 0,
-	})
-	assert.Error(t, err)
-
-	expr = `exists Int64Field `
+	expr = `exists JSONField["A"]["B"]["C"] `
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
 		MetricType:   "",
 		SearchParams: "",
 		RoundDecimal: 0,
 	})
-	assert.Error(t, err)
+	assert.NoError(t, err)
 
-	expr = `exists JSONField["A"]["B"]["C"] `
+	expr = `A["B"][0] > 100`
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
 		MetricType:   "",
@@ -831,7 +843,7 @@ func Test_JSONExpr(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	expr = `A[B][0] > 100`
+	expr = `JSONField[0] > 100`
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
 		MetricType:   "",
@@ -840,7 +852,7 @@ func Test_JSONExpr(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	expr = `A['B'][0] > 100`
+	expr = `A["\"\"B\"\""] > 10`
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
 		MetricType:   "",
@@ -849,7 +861,7 @@ func Test_JSONExpr(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	expr = `JSONField[0] > 100`
+	expr = `A["[\"B\"]"] == "abc\"bbb\"cc"`
 	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
 		Topk:         0,
 		MetricType:   "",
@@ -934,6 +946,60 @@ func Test_InvalidExprOnJSONField(t *testing.T) {
 		RoundDecimal: 0,
 	})
 	assert.Error(t, err)
+
+	expr = `exists JSONField["A"] > 10 `
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `exists Int64Field `
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `A[[""B""]] > 10`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `A["[""B""]"] > 10`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `A[[""B""]] > 10`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `A[B] > 10`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
 }
 
 func Test_InvalidExprWithoutJSONField(t *testing.T) {

internal/parser/planparserv2/pool_test.go

@@ -3,11 +3,9 @@ package planparserv2
 import (
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/antlr/antlr4/runtime/Go/antlr"
-
 	antlrparser "github.com/milvus-io/milvus/internal/parser/planparserv2/generated"
+	"github.com/stretchr/testify/assert"
 )
 
 func genNaiveInputStream() *antlr.InputStream {

pkg/util/typeutil/schema.go

@@ -23,9 +23,8 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
-	"go.uber.org/zap"
-
 	"github.com/milvus-io/milvus/pkg/log"
+	"go.uber.org/zap"
 )
 
 const DynamicFieldMaxLength = 512
@@ -249,6 +248,7 @@ func (helper *SchemaHelper) getDefaultJSONField() (*schemapb.FieldSchema, error)
 	var field *schemapb.FieldSchema
 	for _, f := range helper.schema.GetFields() {
 		// TODO @xiaocai2333: get $SYS_META json field
+		//if f.DataType == schemapb.DataType_JSON && f.GetIsDynamic() {
 		if f.DataType == schemapb.DataType_JSON {
 			if field != nil {
 				// TODO @xiaocai2333: will not return error after support $SYS_META
@@ -260,7 +260,7 @@ func (helper *SchemaHelper) getDefaultJSONField() (*schemapb.FieldSchema, error)
 		}
 	}
 	if field == nil {
-		errMsg := "there is no json field in schema, need to specified field name"
+		errMsg := "there is no dynamic json field in schema, need to specified field name"
 		log.Warn(errMsg)
 		return nil, fmt.Errorf(errMsg)
 	}

tests/integration/json_expr_test.go

@@ -84,6 +84,7 @@ func TestJsonExpr(t *testing.T) {
 			Name:        jsonField,
 			Description: "this is a json field",
 			DataType:    schemapb.DataType_JSON,
+			//IsDynamic:   true,
 		}
 		return &schemapb.CollectionSchema{
 			Name:        collectionName,