Support single quotes with string (#24386)

Signed-off-by: Ncai.zhang <cai.zhang@zilliz.com>

internal/parser/planparserv2/Plan.g4

@@ -84,14 +84,16 @@ FloatingConstant:
 
 Identifier: Nondigit (Nondigit | Digit)* | '$meta';
 
-StringLiteral: EncodingPrefix? '"' SCharSequence? '"';
+StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
 JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
 
 fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
 
-fragment SCharSequence: SChar+;
+fragment DoubleSCharSequence: DoubleSChar+;
+fragment SingleSCharSequence: SingleSChar+;
 
-fragment SChar: ~["\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
+fragment DoubleSChar: ~["\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
+fragment SingleSChar: ~['\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
 fragment Nondigit: [a-zA-Z_];
 fragment Digit: [0-9];
 fragment BinaryConstant: '0' [bB] [0-1]+;

internal/parser/planparserv2/parser_visitor.go

@@ -125,9 +125,10 @@ func (v *ParserVisitor) VisitFloating(ctx *parser.FloatingContext) interface{} {
 
 // VisitString translates expr to GenericValue.
 func (v *ParserVisitor) VisitString(ctx *parser.StringContext) interface{} {
-	literal, err := strconv.Unquote(ctx.StringLiteral().GetText())
-	if err != nil {
-		return err
+	literal := ctx.StringLiteral().GetText()
+	if (strings.HasPrefix(literal, "\"") && strings.HasSuffix(literal, "\"")) ||
+		(strings.HasPrefix(literal, "'") && strings.HasSuffix(literal, "'")) {
+		literal = literal[1 : len(literal)-1]
 	}
 	return &ExprWithType{
 		dataType: schemapb.DataType_VarChar,
@@ -418,9 +419,10 @@ func (v *ParserVisitor) VisitLike(ctx *parser.LikeContext) interface{} {
 		return fmt.Errorf("like operation on complicated expr is unsupported")
 	}
 
-	pattern, err := strconv.Unquote(ctx.StringLiteral().GetText())
-	if err != nil {
-		return err
+	pattern := ctx.StringLiteral().GetText()
+	if (strings.HasPrefix(pattern, "\"") && strings.HasSuffix(pattern, "\"")) ||
+		(strings.HasPrefix(pattern, "'") && strings.HasSuffix(pattern, "'")) {
+		pattern = pattern[1 : len(pattern)-1]
 	}
 
 	op, operand, err := translatePatternMatch(pattern)

internal/parser/planparserv2/plan_parser_v2.go

@@ -3,9 +3,12 @@ package planparserv2
 import (
 	"fmt"
 
+	"go.uber.org/zap"
+
 	"github.com/antlr/antlr4/runtime/Go/antlr"
 	"github.com/milvus-io/milvus-proto/go-api/schemapb"
 	"github.com/milvus-io/milvus/internal/proto/planpb"
+	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
 )
 
@@ -32,6 +35,11 @@ func handleExpr(schema *typeutil.SchemaHelper, exprStr string) interface{} {
 		return errorListener.err
 	}
 
+	if parser.GetCurrentToken().GetTokenType() != antlr.TokenEOF {
+		log.Info("invalid expression", zap.String("expr", exprStr))
+		return fmt.Errorf("invalid expression: %s", exprStr)
+	}
+
 	// lexer & parser won't be used by this thread, can be put into pool.
 	putLexer(lexer)
 	putParser(parser)

internal/parser/planparserv2/plan_parser_v2_test.go

@@ -1148,3 +1148,62 @@ func Test_InvalidExprWithMultipleJSONField(t *testing.T) {
 	})
 	assert.Error(t, err)
 }
+
+func Test_exprWithSingleQuotes(t *testing.T) {
+	schema := newTestSchema()
+	expr := ""
+	var err error
+	expr = `'abc' < StringField < "def"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
+	expr = `'ab"c' < StringField < "d'ef"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
+	expr = `'ab\"c' < StringField < "d\'ef"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
+	expr = `'ab\'c' < StringField < "d\"ef"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.NoError(t, err)
+
+	expr = `'abc'd' < StringField < "def"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+
+	expr = `'abc' < StringField < "def"g"`
+	_, err = CreateSearchPlan(schema, expr, "FloatVectorField", &planpb.QueryInfo{
+		Topk:         0,
+		MetricType:   "",
+		SearchParams: "",
+		RoundDecimal: 0,
+	})
+	assert.Error(t, err)
+}

tests/integration/jsonexpr/json_expr_test.go

@@ -615,6 +615,40 @@ func (s *JSONExprSuite) checkSearch(collectionName, fieldName string, dim int) {
 	s.doSearch(collectionName, []string{fieldName}, expr, dim, checkFunc)
 	log.Info("like expression run successfully")
 
+	expr = `str1 like 'abc\"def-%'`
+	checkFunc = func(result *milvuspb.SearchResults) {
+		s.Equal(1, len(result.Results.FieldsData))
+		s.Equal(fieldName, result.Results.FieldsData[0].GetFieldName())
+		s.Equal(schemapb.DataType_JSON, result.Results.FieldsData[0].GetType())
+		s.Equal(10, len(result.Results.FieldsData[0].GetScalars().GetJsonData().GetData()))
+	}
+	s.doSearch(collectionName, []string{fieldName}, expr, dim, checkFunc)
+	log.Info("like expression run successfully")
+
+	expr = `str1 like 'abc"def-%'`
+	checkFunc = func(result *milvuspb.SearchResults) {
+		s.Equal(0, len(result.Results.FieldsData))
+	}
+	s.doSearch(collectionName, []string{fieldName}, expr, dim, checkFunc)
+	log.Info("like expression run successfully")
+
+	expr = `str2 like 'abc\"def-%'`
+	checkFunc = func(result *milvuspb.SearchResults) {
+		s.Equal(0, len(result.Results.FieldsData))
+	}
+	s.doSearch(collectionName, []string{fieldName}, expr, dim, checkFunc)
+	log.Info("like expression run successfully")
+
+	expr = `str2 like 'abc"def-%'`
+	checkFunc = func(result *milvuspb.SearchResults) {
+		s.Equal(1, len(result.Results.FieldsData))
+		s.Equal(fieldName, result.Results.FieldsData[0].GetFieldName())
+		s.Equal(schemapb.DataType_JSON, result.Results.FieldsData[0].GetType())
+		s.Equal(10, len(result.Results.FieldsData[0].GetScalars().GetJsonData().GetData()))
+	}
+	s.doSearch(collectionName, []string{fieldName}, expr, dim, checkFunc)
+	log.Info("like expression run successfully")
+
 	expr = `A in []`
 	checkFunc = func(result *milvuspb.SearchResults) {
 		s.Equal(0, len(result.Results.FieldsData))
@@ -810,6 +844,8 @@ func newJSONData(fieldName string, rowNum int) *schemapb.FieldData {
 				"F": i,
 				"G": i + 10,
 			},
+			"str1": `abc\"def-` + string(rune(i)),
+			"str2": fmt.Sprintf("abc\"def-%d", i),
 		}
 		if i%2 == 0 {
 			data = map[string]interface{}{