Merge pull request #8441 from taosdata/feature/TD-10708

Feature/td 10708 regex match support for nchar

src/client/src/tscSQLParser.c

@@ -4523,10 +4523,8 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
 // check for match expression
 static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
   const char* msg1 = "regular expression string should be less than %d characters";
-  const char* msg2 = "illegal column type for match/nmatch";
   const char* msg3 = "invalid regular expression";
 
-  tSqlExpr* pLeft  = pExpr->pLeft;
   tSqlExpr* pRight = pExpr->pRight;
 
   if (pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) {
@@ -4536,11 +4534,6 @@ static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_
       return invalidOperationMsg(msgBuf, tmp);
     }
 
-    SSchema* pSchema = tscGetTableSchema(pTableMeta);
-    if ((!isTablenameToken(&pLeft->columnName)) &&(pSchema[index].type != TSDB_DATA_TYPE_BINARY)) {
-      return invalidOperationMsg(msgBuf, msg2);
-    }
-
     if (!(pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_BINARY)) {
       return invalidOperationMsg(msgBuf, msg3);
     }

src/query/src/qFilter.c

@@ -1841,6 +1841,15 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
       qError("dump value to type[%d] failed", type);
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
+
+    // match/nmatch for nchar type need convert from ucs4 to mbs
+    if(type == TSDB_DATA_TYPE_NCHAR &&
+        (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
+      char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
+      int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
+      varDataSetLen(newValData, len);
+      varDataCopy(fi->data, newValData);
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2960,9 +2969,18 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat
       all = false;
       continue;
     }
+    // match/nmatch for nchar type need convert from ucs4 to mbs
+
+    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
+      char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+      int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+      varDataSetLen(newColData, len);
+      (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
+      tfree(newColData);
+    }else{
+      (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
+    }
 
-    (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
-            
     if ((*p)[i] == 0) {
       all = false;
     }
@@ -3009,7 +3027,15 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
             } else if (cunit->rfunc >= 0) {
               (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]);
             } else {
-              (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
+              if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
+                char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+                int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+                varDataSetLen(newColData, len);
+                (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
+                tfree(newColData);
+              }else{
+                (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
+              }
             }
           
           //FILTER_UNIT_SET_R(info, uidx, p[i]);

tests/pytest/query/queryRegex.py

@@ -29,18 +29,18 @@ class TDTestCase:
         print("==============step1")
         ##2021-09-17 For jira: https://jira.taosdata.com:18080/browse/TD-6585
         tdSql.execute(
-            "create stable if not exists stb_test(ts timestamp,c0 binary(32),c1 int) tags(t0 binary(32))"
+            "create stable if not exists stb_test(ts timestamp,c0 binary(32),c1 int,c2 nchar(50)) tags(t0 binary(32),t1 nchar(50))"
         )
         tdSql.execute(
-            'create table if not exists stb_1 using stb_test tags("abcdefgasdfg12346")'
+            'create table if not exists stb_1 using stb_test tags("abcdefgasdfg12346","涛思数据")'
         )
-        tdLog.info('insert into stb_1 values("2021-09-13 10:00:00.001","abcefdasdqwerxasdazx12345",15')
+        tdLog.info('insert into stb_1 values("2021-09-13 10:00:00.001","abcefdasdqwerxasdazx12345",15,"引擎一组"')
 
 
-        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.002","abcefdasdqwerxasdazx12345",15)')
-        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.003","aaaaafffwwqqxzz",16)')
-        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.004","fffwwqqxzz",17)')
-        tdSql.execute('insert into stb_1 values("2020-10-13 10:00:00.001","abcd\\\efgh",100)')
+        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.002","abcefdasdqwerxasdazx12345",15,"引擎一组一号")')
+        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.003","aaaaafffwwqqxzz",16,"引擎一组二号")')
+        tdSql.execute('insert into stb_1 values("2021-09-13 10:00:00.004","fffwwqqxzz",17,"涛涛思思")')
+        tdSql.execute('insert into stb_1 values("2020-10-13 10:00:00.001","abcd\\\efgh",100,"思涛思")')
 
         tdSql.query('select * from stb_test where tbname match "asd"')
         tdSql.checkRows(0)
@@ -98,6 +98,31 @@ class TDTestCase:
         tdSql.query("select * from stb_1 where c0 nmatch '\\\\'")
         tdSql.checkRows(3)
 
+        #2021-10-20 for https://jira.taosdata.com:18080/browse/TD-10708
+        tdSql.query('select * from stb_1 where c2 match "^涛"')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from stb_1 where c2 nmatch "^涛"')
+        tdSql.checkRows(3)
+
+        tdSql.query('select * from stb_1 where c2 match "号$"')
+        tdSql.checkRows(2)
+
+        tdSql.query('select * from stb_1 where c2 nmatch "号$"')
+        tdSql.checkRows(2)
+
+        tdSql.query('select * from stb_1 where c2 match "涛+思"')
+        tdSql.checkRows(2)
+
+        tdSql.query('select * from stb_1 where c2 nmatch "涛+思"')
+        tdSql.checkRows(2)
+
+        tdSql.query('select * from stb_1 where c2 match "涛*思"')
+        tdSql.checkRows(2)
+
+        tdSql.query('select * from stb_1 where c2 nmatch "涛*思"')
+        tdSql.checkRows(2)
+