fix: add limit for json operator

include/util/taoserror.h

@@ -387,6 +387,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_TASK_MSG_ERROR            TAOS_DEF_ERROR_CODE(0, 0x0719)
 #define TSDB_CODE_QRY_JOB_FREED                 TAOS_DEF_ERROR_CODE(0, 0x071A)
 #define TSDB_CODE_QRY_TASK_STATUS_ERROR         TAOS_DEF_ERROR_CODE(0, 0x071B)
+//json
+#define TSDB_CODE_QRY_JSON_IN_ERROR             TAOS_DEF_ERROR_CODE(0, 0x071C)
+#define TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR    TAOS_DEF_ERROR_CODE(0, 0x071D)
+#define TSDB_CODE_QRY_JSON_IN_GROUP_ERROR       TAOS_DEF_ERROR_CODE(0, 0x071E)
 
 // grant
 #define TSDB_CODE_GRANT_EXPIRED                 TAOS_DEF_ERROR_CODE(0, 0x0800)

source/common/src/tdatablock.c

@@ -716,7 +716,12 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
 
     void* left1 = colDataGetData(pColInfoData, left);
     void* right1 = colDataGetData(pColInfoData, right);
-
+    if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+      if (tTagIsJson(left1) || tTagIsJson(right1)) {
+        terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+        return 0;
+      }
+    }
     __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
 
     int ret = fn(left1, right1);
@@ -890,7 +895,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
         SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
 
         int64_t p0 = taosGetTimestampUs();
-
+        
         __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
         qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
 
@@ -919,6 +924,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
   }
 
   taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar);
+  if(terrno) return terrno;
 
   int64_t p1 = taosGetTimestampUs();
 

source/libs/executor/src/executil.c

@@ -271,6 +271,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
   SNode*  pNew = NULL;
   int32_t code = scalarCalculateConstants(pTagCondTmp, &pNew);
   if (TSDB_CODE_SUCCESS != code) {
+    terrno = code;
     nodesDestroyNode(pTagCondTmp);
     return false;
   }
@@ -323,11 +324,18 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
       code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
     }
 
+    if (code != TSDB_CODE_SUCCESS) {
+      qError("failed  to  get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid);
+      terrno = code;
+      return code;
+    }
+
     if(pTagCond){
       int32_t i = 0;
       while(i < taosArrayGetSize(pListInfo->pTableList)) {
         STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
         bool isOk = isTableOk(info, pTagCond, metaHandle);
+        if(terrno) return terrno;
         if(!isOk){
           taosArrayRemove(pListInfo->pTableList, i);
           continue;

source/libs/executor/src/executorimpl.c

@@ -4013,6 +4013,12 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
         isNull[index++] = 0;
         char*       data = nodesGetValueFromNode(pValue);
         if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
+          if(tTagIsJson(data)){
+            terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
+            taosMemoryFree(keyBuf);
+            nodesClearList(groupNew);
+            return terrno;
+          }
           int32_t len = getJsonValueLen(data);
           memcpy(pStart, data, len);
           pStart += len;
@@ -4070,6 +4076,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
       if(code){
+        pTaskInfo->code = code;
         return NULL;
       }
       code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
@@ -4095,7 +4102,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
             .maxTs = INT64_MIN,
       };
       if (pHandle) {
-        createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
+        int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
+        if(code){
+          pTaskInfo->code = code;
+          return NULL;
+        }
       }
 
       SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);

source/libs/executor/src/groupoperator.c

@@ -141,6 +141,10 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
       pkey->isNull = false;
       char* val = colDataGetData(pColInfoData, rowIndex);
       if (pkey->type == TSDB_DATA_TYPE_JSON) {
+        if(tTagIsJson(val)){
+          terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
+          return;
+        }
         int32_t dataLen = getJsonValueLen(val);
         memcpy(pkey->pData, val, dataLen);
       } else if (IS_VAR_DATA_TYPE(pkey->type)) {
@@ -227,11 +231,15 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
   int32_t     len = 0;
   STimeWindow w = TSWINDOW_INITIALIZER;
 
+  terrno = TSDB_CODE_SUCCESS;
   int32_t num = 0;
   for (int32_t j = 0; j < pBlock->info.rows; ++j) {
     // Compare with the previous row of this column, and do not set the output buffer again if they are identical.
     if (!pInfo->isInit) {
       recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j);
+      if (terrno != TSDB_CODE_SUCCESS) {  // group by json error
+        longjmp(pTaskInfo->env, terrno);
+      }
       pInfo->isInit = true;
       num++;
       continue;
@@ -247,6 +255,9 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     if (j == 0) {
       num++;
       recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j);
+      if (terrno != TSDB_CODE_SUCCESS) {  // group by json error
+        longjmp(pTaskInfo->env, terrno);
+      }
       continue;
     }
 
@@ -661,7 +672,11 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
       }
     }
 
+    terrno = TSDB_CODE_SUCCESS;
     doHashPartition(pOperator, pBlock);
+    if (terrno != TSDB_CODE_SUCCESS) {  // group by json error
+      longjmp(pTaskInfo->env, terrno);
+    }
   }
 
   SArray* groupArray = taosArrayInit(taosHashGetSize(pInfo->pGroupSet), sizeof(SDataGroupInfo));

source/libs/executor/src/tsort.c

@@ -593,7 +593,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
       if (size > sortBufSize) {
         // Perform the in-memory sort and then flush data in the buffer into disk.
         int64_t p = taosGetTimestampUs();
-        blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+        code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+        if (code != 0) {
+          return code;
+        }
 
         int64_t el = taosGetTimestampUs() - p;
         pHandle->sortElapsed += el;
@@ -608,7 +611,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
       // Perform the in-memory sort and then flush data in the buffer into disk.
       int64_t p = taosGetTimestampUs();
 
-      blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+      int32_t code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+      if (code != 0) {
+        return code;
+      }
 
       int64_t el = taosGetTimestampUs() - p;
       pHandle->sortElapsed += el;

source/libs/scalar/src/filter.c

@@ -192,6 +192,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
       case TSDB_DATA_TYPE_DOUBLE:        
       case TSDB_DATA_TYPE_TIMESTAMP:        
         return 18;
+      case TSDB_DATA_TYPE_JSON:
+        terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
+        return 0;
       default:
         assert(0);
     }
@@ -215,6 +218,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
       case TSDB_DATA_TYPE_DOUBLE:        
       case TSDB_DATA_TYPE_TIMESTAMP:        
         return 24;
+      case TSDB_DATA_TYPE_JSON:
+        terrno = TSDB_CODE_QRY_JSON_IN_ERROR;
+        return 0;
       default:
         assert(0);
     }

source/libs/scalar/src/scalar.c

@@ -551,7 +551,9 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
   SScalarParam* pLeft = &params[0];
   SScalarParam* pRight = paramNum > 1 ? &params[1] : NULL;
 
+  terrno = TSDB_CODE_SUCCESS;
   OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC);
+  code = terrno;
 
 _return:
   for (int32_t i = 0; i < paramNum; ++i) {

source/libs/scalar/src/sclvector.c

@@ -160,6 +160,9 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){
     return 0;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
     convertNcharToDouble(data+CHAR_BYTES, &out);
+  } else if(tTagIsJson(data)){
+    terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+    return 0;
   } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
@@ -416,6 +419,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
       else if(*data == TSDB_DATA_TYPE_NCHAR) {
         data += CHAR_BYTES;
         convertType = TSDB_DATA_TYPE_NCHAR;
+      } else if(tTagIsJson(data)){
+        terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+        return terrno;
       } else {
         convertNumberToNumber(data+CHAR_BYTES, colDataGetNumData(pOut->columnData, i), *data, outType);
         continue;
@@ -461,7 +467,10 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
     return out;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
     convertNcharToDouble(data+CHAR_BYTES, &out);
-  } else {
+  } else if(tTagIsJson(data)){
+    terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+    return 0;
+  } else{
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
   return out;
@@ -493,10 +502,18 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
   }
 
   if(typeLeft == TSDB_DATA_TYPE_JSON){
+    if(tTagIsJson(*pLeftData)){
+      terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+      return false;
+    }
     typeLeft = **pLeftData;
     (*pLeftData) ++;
   }
   if(typeRight == TSDB_DATA_TYPE_JSON){
+    if(tTagIsJson(*pLeftData)){
+      terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
+      return false;
+    }
     typeRight = **pRightData;
     (*pRightData) ++;
   }
@@ -1576,7 +1593,11 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) {
   int32_t       i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
   int32_t       step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+
   __compar_fn_t fp = filterGetCompFunc(GET_PARAM_TYPE(pLeft), optr);
+  if(terrno != TSDB_CODE_SUCCESS){
+    return;
+  }
 
   pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
 
@@ -1709,6 +1730,7 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
 STagVal getJsonValue(char *json, char *key, bool *isExist) {
   STagVal val = {.pKey = key};
   if (tTagIsJson((const STag *)json) == false){
+    terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
     if(isExist){
       *isExist = false;
     }

source/util/src/terror.c

@@ -394,6 +394,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUPLICATTED_OPERATION,    "Duplicatted operation
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR,           "Task message error")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED,                "Job already freed")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_STATUS_ERROR,        "Task status error")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_IN_ERROR,            "Json not support in in/notin operator")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR,   "Json not support in this place")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_IN_GROUP_ERROR,      "Json not support in group/partition by")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                "License expired")

tests/system-test/2-query/json_tag.py

@@ -213,9 +213,19 @@ class TDTestCase:
 
         # test where with json tag
         tdSql.query("select * from jsons1_1 where jtag is not null")
-        # tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'")
+        tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'")
         tdSql.error("select * from jsons1 where jtag->'tag1'={}")
 
+        # test json error
+        tdSql.error("select jtag + 1 from jsons1")
+        tdSql.error("select jtag > 1 from jsons1")
+        tdSql.error("select jtag like \"1\" from jsons1")
+        tdSql.error("select jtag in  (\"1\") from jsons1")
+        tdSql.error("select jtag from jsons1 where jtag > 1")
+        tdSql.error("select jtag from jsons1 where jtag like 'fsss'")
+        tdSql.error("select jtag from jsons1 where jtag in (1)")
+
+
         # where json value is string
         tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'")
         tdSql.checkRows(2)
@@ -364,7 +374,7 @@ class TDTestCase:
         tdSql.checkRows(2)
 
         # test where condition in  no support in
-        # tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')")
+        tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')")
 
         # test where condition match/nmath
         tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'")
@@ -382,8 +392,8 @@ class TDTestCase:
         tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')")
         tdSql.query("select distinct jtag->'tag1' from jsons1")
         tdSql.checkRows(8)
-        tdSql.query("select distinct jtag from jsons1")
-        tdSql.checkRows(9)
+        # tdSql.query("select distinct jtag from jsons1")
+        # tdSql.checkRows(9)
 
         #test dumplicate key with normal colomn
         tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")")
@@ -419,9 +429,9 @@ class TDTestCase:
         tdSql.checkData(7, 1, "false")
 
 
-        # tdSql.error("select count(*) from jsons1 group by jtag")
-        # tdSql.error("select count(*) from jsons1 partition by jtag")
-        # tdSql.error("select count(*) from jsons1 group by jtag order by jtag")
+        tdSql.error("select count(*) from jsons1 group by jtag")
+        tdSql.error("select count(*) from jsons1 partition by jtag")
+        tdSql.error("select count(*) from jsons1 group by jtag order by jtag")
         tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'")
         tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag")
         tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc")
@@ -487,6 +497,7 @@ class TDTestCase:
         tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}')
 
         tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)")
+        tdSql.error("select t->'tag1' from (select jtag->'tag1' as t, dataint from jsons1)")
         # tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)")
         # tdSql.checkRows(11)
         # tdSql.checkData(1, 1, "jsons1_1")