Merge pull request #13483 from taosdata/feature/TD-13041

feat: add sort/group logic for json

include/common/tdatablock.h

@@ -71,20 +71,14 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
 #define colDataGetData(p1_, r_) \
   ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
 
-static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
-  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
-    if (colDataIsNull_var(pColumnInfoData, row)) {
-      return true;
-    }
-    char* data = colDataGetVarData(pColumnInfoData, row);
-    return (*data == TSDB_DATA_TYPE_NULL);
-  }
+#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL)
 
+static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
   if (!pColumnInfoData->hasNull) {
     return false;
   }
 
-  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {

include/util/tdef.h

@@ -129,7 +129,7 @@ typedef enum EOperatorType {
   OP_TYPE_SUB,
   OP_TYPE_MULTI,
   OP_TYPE_DIV,
-  OP_TYPE_MOD,
+  OP_TYPE_REM,
   // unary arithmetic operator
   OP_TYPE_MINUS,
   OP_TYPE_ASSIGN,

source/common/src/tdatablock.c

@@ -109,7 +109,7 @@ int32_t getJsonValueLen(const char *data) {
     dataLen = DOUBLE_BYTES + CHAR_BYTES;
   } else if (*data == TSDB_DATA_TYPE_BOOL) {
     dataLen = CHAR_BYTES + CHAR_BYTES;
-  } else if (*data == TD_TAG_JSON) {   // json string
+  } else if (*data & TD_TAG_JSON) {   // json string
     dataLen = ((STag*)(data))->len;
   } else {
     ASSERT(0);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1096,7 +1096,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp) {
     case OP_TYPE_SUB:
     case OP_TYPE_MULTI:
     case OP_TYPE_DIV:
-    case OP_TYPE_MOD:
+    case OP_TYPE_REM:
       return true;
     default:
       break;

source/libs/parser/inc/sql.y

@@ -611,7 +611,7 @@ expression(A) ::= expression(B) NK_SLASH expression(C).
 expression(A) ::= expression(B) NK_REM expression(C).                             {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, C);
-                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
                                                                                   }
 expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C).                      {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);

source/libs/parser/src/sql.c

@@ -4079,7 +4079,7 @@ static YYACTIONTYPE yy_reduce(
 {
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172);
                                                                                     SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172);
-                                                                                    yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); 
+                                                                                    yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172)));
                                                                                   }
   yymsp[-2].minor.yy172 = yylhsminor.yy172;
         break;

source/libs/scalar/inc/filterInt.h

@@ -350,7 +350,7 @@ struct SFilterInfo {
 
 extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
 extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
-
+extern OptrStr gOptrStr[];
 
 #ifdef __cplusplus
 }

source/libs/scalar/src/filter.c

@@ -29,8 +29,9 @@ OptrStr gOptrStr[] = {
   {OP_TYPE_SUB,                            "-"},
   {OP_TYPE_MULTI,                          "*"},
   {OP_TYPE_DIV,                            "/"},
-  {OP_TYPE_MOD,                            "%"},
-
+  {OP_TYPE_REM,                            "%"},
+  {OP_TYPE_MINUS,                          "minus"},
+  {OP_TYPE_ASSIGN,                         "assign"},
   // bit operator
   {OP_TYPE_BIT_AND,                        "&"},
   {OP_TYPE_BIT_OR,                         "|"},

tests/system-test/2-query/json_tag.py

@@ -137,9 +137,9 @@ class TDTestCase:
         tdSql.checkRows(9)
         tdSql.query("select jtag from jsons1")
         tdSql.checkRows(13)
-        # tdSql.query("select jtag from jsons1 where jtag is null")
+        tdSql.query("select jtag from jsons1 where jtag is null")
         # tdSql.checkRows(5)
-        # tdSql.query("select jtag from jsons1 where jtag is not null")
+        tdSql.query("select jtag from jsons1 where jtag is not null")
         # tdSql.checkRows(8)
 
         # test jtag is NULL
@@ -260,9 +260,9 @@ class TDTestCase:
         # tdSql.checkRows(1)
         #
         # # where json is null
-        # tdSql.query("select * from jsons1 where jtag is null")
+        tdSql.query("select * from jsons1 where jtag is null")
         # tdSql.checkRows(1)
-        # tdSql.query("select * from jsons1 where jtag is not null")
+        tdSql.query("select * from jsons1 where jtag is not null")
         # tdSql.checkRows(8)
         #
         # # where json key is null
@@ -389,8 +389,8 @@ class TDTestCase:
         tdSql.checkData(2, 1, "11.000000000")
         tdSql.checkData(5, 0, 1)
         tdSql.checkData(5, 1, "false")
-        # tdSql.checkData(6, 0, 1)
-        # tdSql.checkData(6, 1, "null")
+        tdSql.checkData(6, 0, 1)
+        tdSql.checkData(6, 1, "null")
         tdSql.checkData(7, 0, 2)
         tdSql.checkData(7, 1, None)
 
@@ -409,7 +409,7 @@ class TDTestCase:
         tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
         tdSql.checkRows(8)
         tdSql.checkData(0, 0, 10)
-        # tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 1, None)
         tdSql.checkData(4, 0, 0)
         tdSql.checkData(4, 1, "5.000000000")
         tdSql.checkData(7, 0, 11)
@@ -424,10 +424,10 @@ class TDTestCase:
         # test top/bottom with group by json tag
         tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'")
         tdSql.checkRows(11)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(2, 0, 4)
         tdSql.checkData(3, 0, 3)
         tdSql.checkData(3, 1, "false")
-        # tdSql.checkData(3, 0, 24)
-        # tdSql.checkData(3, 1, None)
         tdSql.checkData(10, 0, 23)
         tdSql.checkData(10, 1, '"femail"')
 
@@ -436,7 +436,7 @@ class TDTestCase:
         # tdSql.checkRows(2)
 
         # subquery with json tag
-        tdSql.query("select * from (select jtag, dataint from jsons1)")
+        tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint")
         tdSql.checkRows(11)
         tdSql.checkData(1, 1, 1)
         tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}')