TD-6129<feature> add tag-> where logic

src/client/src/tscSQLParser.c

@@ -4416,6 +4416,37 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+// check for match expression
+static int32_t validateJsonTagExpr(tSqlExpr* pExpr, char* msgBuf) {
+  const char* msg1 = "not support json tag column filter";
+  const char* msg2 = "tag json key is too long, exceed to 64";
+  const char* msg3 = "tag json key must be string";
+
+  tSqlExpr* pLeft = pExpr->pLeft;
+  tSqlExpr* pRight = pExpr->pRight;
+
+  if (pExpr->tokenId == TK_QUESTION) {
+    if (pRight != NULL && !IS_VAR_DATA_TYPE(pRight->value.nType))
+      return invalidOperationMsg(msgBuf, msg3);
+
+    if (pRight != NULL && pRight->value.nLen >= TSDB_COL_NAME_LEN)
+      return invalidOperationMsg(msgBuf, msg2);
+  } else {
+    if (pLeft != NULL && (pLeft->tokenId == TK_ID)) {
+      return invalidOperationMsg(msgBuf, msg1);
+    }
+
+    if (pLeft != NULL && (pLeft->tokenId == TK_ARROW)) {
+      if (pLeft->pRight && !IS_VAR_DATA_TYPE(pLeft->pRight->value.nType))
+        return invalidOperationMsg(msgBuf, msg3);
+      if (pLeft->pRight && pLeft->pRight->value.nLen >= TSDB_COL_NAME_LEN)
+        return invalidOperationMsg(msgBuf, msg2);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 // check for match expression
 static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
   const char* msg1 = "regular expression string should be less than %d characters";
@@ -4487,14 +4518,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   const char* msg2 = "illegal column name";
   const char* msg4 = "too many join tables";
   const char* msg5 = "not support ordinary column join";
-  const char* msg6 = "not support json tag column filter";
-  const char* msg7 = "tag json key is too long, exceed to 64";
 
   tSqlExpr* pLeft  = (*pExpr)->pLeft;
   tSqlExpr* pRight = (*pExpr)->pRight;
 
   SStrToken* colName = NULL;
-  if(pLeft->tokenId == TK_ARROW || pLeft->tokenId == TK_QUESTION){
+  if(pLeft->tokenId == TK_ARROW){
     colName = &(pLeft->pLeft->columnName);
   }else{
     colName = &(pLeft->columnName);
@@ -4611,13 +4640,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
 
-    if (pSchema->type == TSDB_DATA_TYPE_JSON && pLeft != NULL && (pLeft->tokenId == TK_ID)){
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
-    }
-
-    if (pSchema->type == TSDB_DATA_TYPE_JSON && pLeft != NULL && (pLeft->tokenId == TK_ARROW)){
-      if(pLeft->pRight && pLeft->pRight->value.nLen >= TSDB_COL_NAME_LEN)
-        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
+    // check for json tag operation -> and ?
+    if (pSchema->type == TSDB_DATA_TYPE_JSON){
+      code = validateJsonTagExpr(*pExpr, tscGetErrorMsgPayload(pCmd));
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
     }
 
     if (pRight != NULL && pRight->tokenId == TK_ID) {  // join on tag columns for stable query
@@ -4639,7 +4667,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
       if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY
                                          && pSchema->type != TSDB_DATA_TYPE_NCHAR
                                          && pSchema->type != TSDB_DATA_TYPE_BOOL)) {
-        handleNeOptr(&rexpr, *pExpr);
+        handleNeOptr(&rexpr, *pExpr);     //todo json check
         *pExpr = rexpr;
       }
       

src/query/src/qFilter.c

@@ -37,6 +37,7 @@ OptrStr gOptrStr[] = {
   {TSDB_RELATION_NOT,                      "not"},
   {TSDB_RELATION_MATCH,                    "match"},
   {TSDB_RELATION_NMATCH,                   "nmatch"},
+  {TSDB_RELATION_QUESTION,                 "?"},
 };
 
 static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) {
@@ -44,7 +45,7 @@ static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const v
   const SSchema *sch2 = desc2;
 
   if(sch1->type == TSDB_DATA_TYPE_JSON && sch2->type == TSDB_DATA_TYPE_JSON){
-    return strcmp(sch1->name, sch2->name);
+    return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId);
   }
   else{
     return sch1->colId != sch2->colId;
@@ -164,7 +165,8 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) {
 __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal,
   compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, 
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
-  setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch
+  setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch,
+  compareStrRegexCompNMatch, comparreStrContainJson
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@@ -207,6 +209,8 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         comparFn = 19;
       } else if (optr == TSDB_RELATION_NMATCH) {
         comparFn = 20;
+      } else if (optr == TSDB_RELATION_QUESTION) {
+        comparFn = 21;
       } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
         comparFn = 7;
       } else if (optr == TSDB_RELATION_IN) {
@@ -223,6 +227,8 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         comparFn = 19;
       } else if (optr == TSDB_RELATION_NMATCH) {
         comparFn = 20;
+      } else if (optr == TSDB_RELATION_QUESTION) {
+        comparFn = 21;
       } else if (optr == TSDB_RELATION_LIKE) {
         comparFn = 9;
       } else if (optr == TSDB_RELATION_IN) {
@@ -853,7 +859,7 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
 }
 
 
-int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
+int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode* parent, tExprNode *node, SFilterFieldId *fid) {
   CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
 
   int32_t type;
@@ -870,6 +876,12 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
     if (node->nodeType == TSQL_NODE_COL) {
       type = FLD_TYPE_COLUMN;
       v = node->pSchema;
+      if(parent->nodeType == TSDB_RELATION_QUESTION){
+        node->pSchema->colId = 0;  // ? operation make colId=0 to make different with -> operation to eliminate repetition and don not convert type
+        assert(parent->_node.pRight->pVal->nLen < TSDB_COL_NAME_LEN);
+        memset(node->pSchema->name, 0, TSDB_COL_NAME_LEN);
+        strncpy(node->pSchema->name, parent->_node.pRight->pVal->pz, parent->_node.pRight->pVal->nLen);
+      }
       node->pSchema = NULL;
     } else {
       type = FLD_TYPE_VALUE;
@@ -1165,7 +1177,7 @@ _return:
 int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) {
   SFilterFieldId left = {0}, right = {0};
 
-  filterAddFieldFromNode(info, tree->_node.pLeft, &left);  
+  filterAddFieldFromNode(info, tree, tree->_node.pLeft, &left);
 
   tVariant* var = tree->_node.pRight->pVal;
   int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
@@ -1220,9 +1232,9 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
 
     taosHashCleanup(data);
   } else {
-    filterAddFieldFromNode(info, tree->_node.pRight, &right);  
+    filterAddFieldFromNode(info, tree, tree->_node.pRight, &right);
     
-    filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);  
+    filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
 
     SFilterGroup fgroup = {0};
     filterAddUnitToGroup(&fgroup, uidx);
@@ -1891,6 +1903,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
     case TSDB_RELATION_MATCH: {
       return ret == 0;
     }
+    case TSDB_RELATION_QUESTION: {
+      return ret == 0;
+    }
     case TSDB_RELATION_NMATCH: {
       return ret == 0;
     }
@@ -2643,9 +2658,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t
       }
     }
 
-    if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL 
+    if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
      || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
-     || cunit->optr == TSDB_RELATION_NOT_EQUAL) {
+     || cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_QUESTION) {
       continue;
     }
 
@@ -3178,7 +3193,7 @@ int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_
 int filterJsonTypeConvert(SFilterInfo* info) {
   for(int i = 0; i < info->fields[FLD_TYPE_COLUMN].num; i++) {
     SSchema* schema = info->fields[FLD_TYPE_COLUMN].fields[i].desc;
-    if(schema->type == TSDB_DATA_TYPE_JSON){
+    if(schema->type == TSDB_DATA_TYPE_JSON && schema->colId != 0){ // schema->colId != 0 means not ? operation
 
       void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name));
       if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
@@ -3188,7 +3203,7 @@ int filterJsonTypeConvert(SFilterInfo* info) {
     }
   }
   for(int i = 0; i < info->unitNum; i++){
-    if(info->units[i].compare.type == TSDB_DATA_TYPE_JSON){
+    if(info->units[i].compare.type == TSDB_DATA_TYPE_JSON && info->units[i].compare.optr == TSDB_RELATION_ARROW){
       SFilterField *colLeft = FILTER_UNIT_LEFT_FIELD(info, &info->units[i]);
       info->units[i].compare.type = FILTER_GET_COL_FIELD_TYPE(colLeft);
 
@@ -3502,7 +3517,7 @@ int32_t filterIsIndexedColumnQuery(SFilterInfo* info, int32_t idxId, bool *res)
   int32_t optr = FILTER_UNIT_OPTR(info->units);
   
   CHK_JMP(optr == TSDB_RELATION_LIKE || optr == TSDB_RELATION_IN || optr == TSDB_RELATION_MATCH 
-       || optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL);
+       || optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == TSDB_RELATION_QUESTION);
 
   *res = true;
 

src/query/src/qSqlParser.c

@@ -328,7 +328,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
     pRSub->Expr.paramList = (SArray *)pRight;
 
     pExpr->pRight = pRSub;
-  } else if (optrType == TK_ARROW) {
+  } else if (optrType == TK_ARROW || optrType == TK_QUESTION) {
     pExpr->tokenId = optrType;
     pExpr->pLeft = pLeft;
     pExpr->pRight = pRight;
@@ -474,8 +474,7 @@ bool tSqlExprIsLeaf(tSqlExpr* pExpr) {
          (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) ||
          (pExpr->tokenId == TK_NULL) ||
          (pExpr->tokenId == TK_SET))) ||
-         (pExpr->tokenId == TK_ARROW)||
-         (pExpr->tokenId == TK_QUESTION);
+         (pExpr->tokenId == TK_ARROW);
 }
 
 bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) {

src/tsdb/src/tsdbRead.c

@@ -4090,7 +4090,7 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){
   for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
     SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
     SSchema*      sch = fi->desc;
-    if (sch-> colId == TSDB_TBNAME_COLUMN_INDEX) continue;
+    if (sch->colId == TSDB_TBNAME_COLUMN_INDEX) continue;
     int32_t outLen = 0;
     char* key = NULL;
     if(JSON_TYPE_NCHAR){

src/util/inc/tcompare.h

@@ -88,6 +88,7 @@ int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
 int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
 int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t comparreStrContainJson(const void* pLeft, const void* pRight);
 
 #ifdef __cplusplus
 }

src/util/src/tcompare.c

@@ -404,6 +404,12 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
   return result;
 }
 
+int32_t comparreStrContainJson(const void* pLeft, const void* pRight) {
+  if(pLeft) return 0;
+  return 1;
+}
+
+
 int32_t taosArrayCompareString(const void* a, const void* b) {
   const char* x = *(const char**)a;
   const char* y = *(const char**)b;
@@ -471,6 +477,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
         comparFn = compareStrRegexCompNMatch;
       } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
         comparFn = compareStrPatternComp;
+      } else if (optr == TSDB_RELATION_QUESTION) {
+        comparFn = comparreStrContainJson;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = compareFindItemInSet;
       } else { /* normal relational comparFn */
@@ -485,6 +493,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
         comparFn = compareStrRegexCompMatch;
       } else if (optr == TSDB_RELATION_NMATCH) {
         comparFn = compareStrRegexCompNMatch;
+      } else if (optr == TSDB_RELATION_QUESTION) {
+        comparFn = comparreStrContainJson;
       } else if (optr == TSDB_RELATION_LIKE) {
         comparFn = compareWStrPatternComp;
       } else if (optr == TSDB_RELATION_IN) {

tests/pytest/stable/json_tag.py

@@ -42,12 +42,22 @@ class TDTestCase:
         tdSql.query("select * from db_json_tag_test.jsons1")
         tdSql.checkRows(1)
 
-        tdSql.query("select jtag from db_json_tag_test.jsons1_1")
+        tdSql.error("select * from db_json_tag_test.jsons1 where jtag->'location'=4")
+
+        tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'location'='beijing'")
         tdSql.checkRows(1)
 
-        tdSql.error("select * from db_json_tag_test.jsons1 where jtag->'location'=4")
+        tdSql.query("select jtag->'location' from db_json_tag_test.jsons1_2")
+        tdSql.checkData(0, 0, "beijing")
+
 
-        tdSql.error("select * from db_json_tag_test.jsons1 where jtag->'location'='beijing'")
+        tdSql.query("select jtag->'num' from db_json_tag_test.jsons1 where jtag->'location'='beijing'")
+        tdSql.checkData(0, 0, 5)
+
+        tdSql.query("select jtag->'location' from db_json_tag_test.jsons1")
+        tdSql.checkRows(2)
+
+        tdSql.query("select jtag from db_json_tag_test.jsons1_1")
         tdSql.checkRows(1)
 
         print("==============step3")
@@ -59,7 +69,7 @@ class TDTestCase:
         tdSql.error("ALTER TABLE db_json_tag_test.jsons1_1 SET TAG jtag=4")
 
         tdSql.execute("ALTER TABLE db_json_tag_test.jsons1_1 SET TAG jtag='{\"sex\":\"femail\",\"age\":35}'")
-        tdSql.query("select jtag from db_json_tag_test.jsons1_1    ")
+        tdSql.query("select jtag from db_json_tag_test.jsons1_1")
         tdSql.checkData(0, 0, "{\"sex\":\"femail\",\"age\":35}")
     def stop(self):
         tdSql.close()