add json tag support for group by

src/client/src/tscSQLParser.c

@@ -3711,8 +3711,6 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
   const char* msg6 = "tags not allowed for table query";
   const char* msg7 = "not support group by expression";
   const char* msg8 = "normal column can only locate at the end of group by clause";
-  const char* msg9 = "json tag format error, like json->'key'";
-  const char* msg10 = "-> operation only apply for json tag";
 
   // todo : handle two tables situation
   STableMetaInfo* pTableMetaInfo = NULL;
@@ -3748,33 +3746,27 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
 
   size_t num = taosArrayGetSize(pList);
   for (int32_t i = 0; i < num; ++i) {
-    tVariantListItem * pItem = taosArrayGet(pList, i);
-    tVariant* pVar = &pItem->pVar;
-
-    SStrToken token = {pVar->nLen, pVar->nType, pVar->pz};
-    char* jsonName = NULL;
-    char* jsonKey = NULL;
-    int ret = getArrowKV(pVar->pz, pVar->nLen, &jsonName, &jsonKey);
-    if(jsonName && jsonKey) tscDebug("group by json k:%s, v:%s", jsonName, jsonKey);
-    if (ret == 2){
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9);
-    }
-    if(ret == 0){
-      token.z = jsonName;
-      token.n = strlen(jsonName);
+    CommonItem * pItem = taosArrayGet(pList, i);
+    SStrToken token = {0};
+    if(pItem->isJsonExp){
+      assert(pItem->jsonExp->tokenId == TK_ARROW);
+      token.n = pItem->jsonExp->pLeft->value.nLen;
+      token.z = pItem->jsonExp->pLeft->value.pz;
+      token.type = pItem->jsonExp->pLeft->value.nType;
+    }else{
+      token.n = pItem->pVar.nLen;
+      token.z = pItem->pVar.pz;
+      token.type = pItem->pVar.nType;
     }
+
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
     if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      tfree(jsonName);
-      tfree(jsonKey);
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
     }
 
     if (tableIndex == COLUMN_INDEX_INITIAL_VAL) {
       tableIndex = index.tableIndex;
     } else if (tableIndex != index.tableIndex) {
-      tfree(jsonName);
-      tfree(jsonKey);
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
     }
 
@@ -3786,18 +3778,12 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
     } else {
       pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
     }
-    if(ret == 0 && pSchema->type != TSDB_DATA_TYPE_JSON){
-      tfree(jsonName);
-      tfree(jsonKey);
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10);
-    }
+
     int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
     bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols);
 
     if (groupTag) {
       if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-        tfree(jsonName);
-        tfree(jsonKey);
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
 
@@ -3807,25 +3793,24 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
       }
 
       SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, };
-      if(ret == 0){
-        tstrncpy(colIndex.name, jsonKey, tListLen(colIndex.name));
+      if(pItem->isJsonExp) {
+        tstrncpy(colIndex.name, pItem->jsonExp->pRight->value.pz, tListLen(colIndex.name));
       }else{
         tstrncpy(colIndex.name, pSchema->name, tListLen(colIndex.name));
       }
+
       taosArrayPush(pGroupExpr->columnInfo, &colIndex);
-      
+
       index.columnIndex = relIndex;
       tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema);
     } else {
       // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by
       if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
-        tfree(jsonName);
-        tfree(jsonKey);
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
       }
 
       tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema);
-      
+
       SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId };
       strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name));
 
@@ -3833,8 +3818,6 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
       pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
       numOfGroupCols++;
     }
-    tfree(jsonName);
-    tfree(jsonKey);
   }
 
   // 1. only one normal column allowed in the group by clause
@@ -3846,7 +3829,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd
   for(int32_t i = 0; i < num; ++i) {
     SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i);
     if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) {
-     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
+      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
     }
   }
 

src/query/inc/qSqlparser.h

@@ -79,6 +79,15 @@ typedef struct tVariantListItem {
   uint8_t            sortOrder;
 } tVariantListItem;
 
+typedef struct CommonItem {
+  union {
+    tVariant           pVar;
+    struct tSqlExpr    *jsonExp;
+  };
+  bool               isJsonExp;
+  uint8_t            sortOrder;
+} CommonItem;
+
 typedef struct SIntervalVal {
   int32_t            token;
   SStrToken          interval;

src/query/inc/sql.y

@@ -621,25 +621,30 @@ sliding_opt(K) ::= .                            {K.n = 0; K.z = NULL; K.type = 0
 %type sortlist {SArray*}
 %destructor sortlist {taosArrayDestroy($$);}
 
-%type sortitem {tVariant}
-%destructor sortitem {tVariantDestroy(&$$);}
-
 orderby_opt(A) ::= .                          {A = 0;}
 orderby_opt(A) ::= ORDER BY sortlist(X).      {A = X;}
 
 sortlist(A) ::= sortlist(X) COMMA item(Y) sortorder(Z). {
-    A = tVariantListAppend(X, &Y, Z);
+  A = commonItemAppend(X, &Y, NULL, false, Z);
+}
+
+sortlist(A) ::= sortlist(X) COMMA arrow(Y) sortorder(Z). {
+  A = commonItemAppend(X, NULL, Y, true, Z);
 }
 
 sortlist(A) ::= item(Y) sortorder(Z). {
-  A = tVariantListAppend(NULL, &Y, Z);
+  A = commonItemAppend(NULL, &Y, NULL, false, Z);
+}
+
+sortlist(A) ::= arrow(Y) sortorder(Z). {
+  A = commonItemAppend(NULL, NULL, Y, true, Z);
 }
 
 %type item {tVariant}
-item(A) ::= ids(X) cpxName(Y).   {
+item(A) ::= ID(X).  { toTSDBType(X.type); tVariantCreate(&A, &X); }
+item(A) ::= ID(X) DOT ID(Y).   {
   toTSDBType(X.type);
   X.n += Y.n;
-
   tVariantCreate(&A, &X);
 }
 
@@ -658,11 +663,19 @@ groupby_opt(A) ::= .                       { A = 0;}
 groupby_opt(A) ::= GROUP BY grouplist(X).  { A = X;}
 
 grouplist(A) ::= grouplist(X) COMMA item(Y).    {
-  A = tVariantListAppend(X, &Y, -1);
+  A = commonItemAppend(X, &Y, NULL, false, -1);
+}
+
+grouplist(A) ::= grouplist(X) COMMA arrow(Y).    {
+  A = commonItemAppend(X, NULL, Y, true, -1);
 }
 
 grouplist(A) ::= item(X).                       {
-  A = tVariantListAppend(NULL, &X, -1);
+  A = commonItemAppend(NULL, &X, NULL, false, -1);
+}
+
+grouplist(A) ::= arrow(X).                       {
+  A = commonItemAppend(NULL, NULL, X, true, -1);
 }
 
 //having clause, ignore the input condition in having
@@ -757,10 +770,15 @@ expr(A) ::= expr(X) MATCH expr(Y).    {A = tSqlExprCreate(X, Y, TK_MATCH);  }
 expr(A) ::= expr(X) NMATCH expr(Y).    {A = tSqlExprCreate(X, Y, TK_NMATCH);  }
 
 // question expression
-expr(A) ::= expr(X) QUESTION expr(Y).    {A = tSqlExprCreate(X, Y, TK_QUESTION);  }
+expr(A) ::= ID(X) QUESTION STRING(Y).    { tSqlExpr* S = tSqlExprCreateIdValue(pInfo, &X, TK_ID); tSqlExpr* M = tSqlExprCreateIdValue(pInfo, &Y, TK_STRING); A = tSqlExprCreate(S, M, TK_QUESTION);  }
 
 // arrow expression
-expr(A) ::= expr(X) ARROW expr(Y).    {A = tSqlExprCreate(X, Y, TK_ARROW);  }
+%type arrow {tSqlExpr*}
+%destructor arrow {tSqlExprDestroy($$);}
+arrow(A) ::= ID(X) ARROW STRING(Y).    {tSqlExpr* S = tSqlExprCreateIdValue(pInfo, &X, TK_ID); tSqlExpr* M = tSqlExprCreateIdValue(pInfo, &Y, TK_STRING); A = tSqlExprCreate(S, M, TK_ARROW);  }
+arrow(A) ::= ID(X) DOT ID(Y) ARROW STRING(Z).    {X.n += (1+Y.n); tSqlExpr* S = tSqlExprCreateIdValue(pInfo, &X, TK_ID); tSqlExpr* M = tSqlExprCreateIdValue(pInfo, &Z, TK_STRING); A = tSqlExprCreate(S, M, TK_ARROW);  }
+
+expr(A) ::= arrow(X). {A = X;}
 
 //in expression
 expr(A) ::= expr(X) IN LP exprlist(Y) RP.   {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); }

src/query/src/qSqlParser.c

@@ -522,6 +522,24 @@ SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order)
   return pList;
 }
 
+SArray *commonItemAppend(SArray *pList, tVariant *pVar, tSqlExpr *jsonExp, bool isJsonExp, uint8_t sortOrder){
+  if (pList == NULL) {
+    pList = taosArrayInit(4, sizeof(CommonItem));
+  }
+
+  CommonItem item;
+  item.sortOrder = sortOrder;
+  item.isJsonExp = isJsonExp;
+  if(isJsonExp){
+    item.jsonExp = jsonExp;
+  }else{
+    item.pVar = *pVar;
+  }
+
+  taosArrayPush(pList, &item);
+  return pList;
+}
+
 SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) {
   if (pList == NULL) {
     pList = taosArrayInit(4, sizeof(tVariantListItem));
@@ -822,6 +840,15 @@ static void freeVariant(void *pItem) {
   tVariantDestroy(&p->pVar);
 }
 
+static void freeCommonItem(void *pItem) {
+  CommonItem* p = (CommonItem *) pItem;
+  if (p->isJsonExp){
+    tSqlExprDestroy(p->jsonExp);
+  }else{
+    tVariantDestroy(&p->pVar);
+  }
+}
+
 void freeCreateTableInfo(void* p) {
   SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p;  
   taosArrayDestroy(pInfo->pTagNames);
@@ -841,10 +868,10 @@ void destroySqlNode(SSqlNode *pSqlNode) {
   tSqlExprDestroy(pSqlNode->pWhere);
   pSqlNode->pWhere = NULL;
   
-  taosArrayDestroyEx(pSqlNode->pSortOrder, freeVariant);
+  taosArrayDestroyEx(pSqlNode->pSortOrder, freeCommonItem);
   pSqlNode->pSortOrder = NULL;
 
-  taosArrayDestroyEx(pSqlNode->pGroupby, freeVariant);
+  taosArrayDestroyEx(pSqlNode->pGroupby, freeCommonItem);
   pSqlNode->pGroupby = NULL;
 
   pSqlNode->from = destroyRelationInfo(pSqlNode->from);

src/util/inc/tutil.h

@@ -38,7 +38,6 @@ char *  strntolower_s(char *dst, const char *src, int32_t n);
 int64_t strnatoi(char *num, int32_t len);
 char *  strbetween(char *string, char *begin, char *end);
 char *  paGetToken(char *src, char **token, int32_t *tokenLen);
-int     getArrowKV(char* src, int32_t len, char** name, char** key);
 
 int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
 int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);

src/util/src/tutil.c

@@ -185,21 +185,6 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
   return NULL;
 }
 
-// src = json->'id'  => name=json key=id
-int getArrowKV(char* src, int32_t len, char** name, char** key){
-  char* loc = strstr(src, "->");
-  if(loc == NULL){return 1;}
-  if(loc-src + 4 >= len) {return 2;}
-  if(*(loc+2) != *(src + len - 1)) {return 2;}
-  if(*(loc+2) != '\'' && *(loc+2) != '"') {return 2;}
-
-  *name = calloc(1, loc-src+1);
-  *key = calloc(1, len - (loc-src+4) + 1);
-  tstrncpy(*name, src, loc-src+1);
-  tstrncpy(*key, loc+2, len - (loc-src+4) + 1);
-  return 0;
-}
-
 char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
   if (!ignoreInEsc) {
     return strstr(src, dst);