TD-6129<feature> optimize json filter logic

src/client/inc/tscUtil.h

@@ -381,7 +381,7 @@ char* parseTagDatatoJson(void *p, uint8_t jsonType);
 void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType);
 
 int8_t  jsonType2DbType(double data, int jsonType, uint8_t type);
-void*      getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType);
+void*      getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* colId);
 
 #ifdef __cplusplus
 }

src/client/src/tscUtil.c

@@ -5177,7 +5177,7 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
 }
 
 void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType){
-  void* result = getJsonTagValue(data, key, keyLen, jsonType);
+  void* result = getJsonTagValue(data, key, keyLen, jsonType, NULL);
   if (result == NULL){    // json key no result
     return;
   }
@@ -5381,11 +5381,14 @@ int8_t jsonType2DbType(double data, int jsonType, uint8_t type){
       if (data - (int64_t)data > 0) return TSDB_DATA_TYPE_DOUBLE; else return TSDB_DATA_TYPE_BIGINT;
     case cJSON_String:
       if (type == TSDB_DATA_TYPE_JSON_NCHAR) return TSDB_DATA_TYPE_NCHAR; else return TSDB_DATA_TYPE_BINARY;
+    case cJSON_True:
+    case cJSON_False:
+      return TSDB_DATA_TYPE_BOOL;
   }
   return TSDB_DATA_TYPE_NULL;
 }
 
-void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType){
+void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* retColId){
   int32_t outLen = 0;
   if(jsonType == TSDB_DATA_TYPE_JSON_NCHAR){
     char tagKey[256] = {0};
@@ -5405,6 +5408,7 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp
     JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ);
     if (p == NULL) return NULL;
     int16_t colId = p->colId + 1;
+    if(retColId) *retColId = p->colId;
     return tdGetKVRowValOfCol(pTable->tagVal, colId);
   }else if(TABLE_TYPE(pTable) == TSDB_SUPER_TABLE){
     SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen);
@@ -5412,6 +5416,7 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp
     if(taosArrayGetSize(*data) == 0) return NULL;
     JsonMapValue* p = taosArrayGet(*data, 0);
     int16_t colId = p->colId + 1;
+    if(retColId) *retColId = p->colId;
     return tdGetKVRowValOfCol(((STable*)(p->table))->tagVal, colId);
   }
   return NULL;

src/common/src/tvariant.c

@@ -847,7 +847,8 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
       break;
     }
     
-    case TSDB_DATA_TYPE_BINARY: {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_JSON_BINARY:{
       if (!includeLengthPrefix) {
         if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
           *(uint8_t*) payload = TSDB_DATA_BINARY_NULL;
@@ -884,7 +885,8 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
       }
       break;
     }
-    case TSDB_DATA_TYPE_NCHAR: {
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON_NCHAR:{
       int32_t newlen = 0;
       if (!includeLengthPrefix) {
         if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
@@ -920,21 +922,21 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
       
       break;
     }
-    case TSDB_DATA_TYPE_JSON_BINARY:
-    case TSDB_DATA_TYPE_JSON_NCHAR:{
-      if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
-        *(int8_t *)payload = TSDB_DATA_TINYINT_NULL;
-      } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){
-        *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
-      } else if (pVariant->nType == TSDB_DATA_TYPE_JSON_BINARY){   // select * from stable, set tag type to json，from setTagValue/tag_project_function
-        memcpy(payload, pVariant->pz, pVariant->nLen);
-      } else if(pVariant->nType == TSDB_DATA_TYPE_JSON_NCHAR){
-        memcpy(payload, pVariant->wpz, pVariant->nLen);
-      }else {
-        return -1;
-      }
-      break;
-    }
+//    case TSDB_DATA_TYPE_JSON_BINARY:
+//    case TSDB_DATA_TYPE_JSON_NCHAR:{
+//      if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
+//        *(int8_t *)payload = TSDB_DATA_TINYINT_NULL;
+//      } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){
+//        *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
+//      } else if (pVariant->nType == TSDB_DATA_TYPE_JSON_BINARY){   // select * from stable, set tag type to json，from setTagValue/tag_project_function
+//        memcpy(payload, pVariant->pz, pVariant->nLen);
+//      } else if(pVariant->nType == TSDB_DATA_TYPE_JSON_NCHAR){
+//        memcpy(payload, pVariant->wpz, pVariant->nLen);
+//      }else {
+//        return -1;
+//      }
+//      break;
+//    }
   }
   
   return 0;

src/query/src/qFilter.c

@@ -858,37 +858,23 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
   return TSDB_CODE_SUCCESS;
 }
 
-
-int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode* parent, tExprNode *node, SFilterFieldId *fid) {
+int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
   CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
+  CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR);
 
   int32_t type;
   void *v;
-  if(node->nodeType == TSQL_NODE_EXPR && node->_node.optr == TSDB_RELATION_ARROW){    // json tag -> operation
-    type = FLD_TYPE_COLUMN;
-    assert(node->_node.pRight->pVal->nLen < TSDB_COL_NAME_LEN);
-    memset(node->_node.pLeft->pSchema->name, 0, TSDB_COL_NAME_LEN);
-    strncpy(node->_node.pLeft->pSchema->name, node->_node.pRight->pVal->pz, node->_node.pRight->pVal->nLen);
-    v = node->_node.pLeft->pSchema;
-    node->_node.pLeft->pSchema = NULL;
-  }else{
-    CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR);
+
   if (node->nodeType == TSQL_NODE_COL) {
     type = FLD_TYPE_COLUMN;
     v = node->pSchema;
-      if(parent->_node.optr == TSDB_RELATION_QUESTION){
-        node->pSchema->colId = 0;  // ? operation make colId=0 to make different with -> operation to eliminate repetition and don not convert type
-        assert(parent->_node.pRight->pVal->nLen < TSDB_COL_NAME_LEN);
-        memset(node->pSchema->name, 0, TSDB_COL_NAME_LEN);
-        strncpy(node->pSchema->name, parent->_node.pRight->pVal->pz, parent->_node.pRight->pVal->nLen);
-      }
     node->pSchema = NULL;
   } else {
     type = FLD_TYPE_VALUE;
     v = node->pVal;
     node->pVal = NULL;
   }
-  }
+
   filterAddField(info, v, NULL, type, fid, 0, true);
   
   return TSDB_CODE_SUCCESS;
@@ -1175,26 +1161,33 @@ _return:
 
 
 int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) {
-  SFilterFieldId left = {0}, right = {0};
-
-  filterAddFieldFromNode(info, tree, tree->_node.pLeft, &left);
-
-  tVariant* var = tree->_node.pRight->pVal;
-  int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
-  int32_t len = 0;
-  uint16_t uidx = 0;
-
-  // if has json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next
+  int32_t type = -1;
   tExprNode* pLeft = tree->_node.pLeft;
   if(pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.optr == TSDB_RELATION_ARROW){    // json tag -> operation
     assert(info->pTable != NULL);
-    SSchema* schema = FILTER_GET_COL_FIELD_DESC(FILTER_GET_FIELD(info, left));
-    void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type);
+    SSchema* schema = pLeft->_node.pLeft->pSchema;
+    void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type, &schema->colId);
     if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
-    type = *(char*)data;
+    type = *(char*)data;  // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next
     assert(type > TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_JSON_BINARY);
+    if(pLeft->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+    memset(schema->name, 0, TSDB_COL_NAME_LEN);
+    strncpy(schema->name, pLeft->_node.pRight->pVal->pz, pLeft->_node.pRight->pVal->nLen);
+    pLeft = pLeft->_node.pLeft;
+  }else if(tree->_node.optr == TSDB_RELATION_QUESTION){
+    SSchema* schema = pLeft->pSchema;
+    if(tree->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+    memset(schema->name, 0, TSDB_COL_NAME_LEN);
+    strncpy(schema->name, tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen);
   }
 
+  SFilterFieldId left = {0}, right = {0};
+  filterAddFieldFromNode(info, pLeft, &left);
+  tVariant* var = tree->_node.pRight->pVal;
+  if(type == -1) type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  int32_t len = 0;
+  uint16_t uidx = 0;
+
   if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) {
     void *data = NULL;
     filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false);
@@ -1235,7 +1228,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
 
     taosHashCleanup(data);
   } else {
-    filterAddFieldFromNode(info, tree, tree->_node.pRight, &right);
+    filterAddFieldFromNode(info, tree->_node.pRight, &right);
     
     filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
 
@@ -1841,10 +1834,10 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
       continue;
     }
 
-    if (type == TSDB_DATA_TYPE_BINARY) {
+    if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON_BINARY) {
       size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
       fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE);
-    } else if (type == TSDB_DATA_TYPE_NCHAR) {
+    } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON_NCHAR) {
       size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;    
       fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
     } else {
@@ -3198,7 +3191,7 @@ int filterJsonTypeConvert(SFilterInfo* info) {
     SSchema* schema = info->fields[FLD_TYPE_COLUMN].fields[i].desc;
     if(IS_JSON_DATA_TYPE(schema->type)){
       if(schema->colId != 0){           // schema->colId != 0 means not ? operation
-        void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type);
+        void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type, NULL);
         if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
         int8_t type = *(char*)data;
         assert(type > TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_JSON_BINARY);
@@ -3248,7 +3241,7 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) {
   ERR_JRET(code);
 
   if(info->pTable){
-    code = filterJsonTypeConvert(info); // convert json type to other type to prepare for th next defination of compare function
+    //code = filterJsonTypeConvert(info); // convert json type to other type to prepare for th next defination of compare function
     ERR_JRET(code);
   }
 

src/tsdb/src/tsdbRead.c

@@ -4081,7 +4081,7 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch
   if (id == TSDB_TBNAME_COLUMN_INDEX) {
     *data = TABLE_NAME(pTable);
   } else {
-    void* jsonData = getJsonTagValue(pTable, name, strlen(name), pTable->pSuper->tagSchema->columns->type);
+    void* jsonData = getJsonTagValue(pTable, name, strlen(name), pTable->pSuper->tagSchema->columns->type, NULL);
     if (jsonData != NULL) jsonData += CHAR_BYTES;   // jump type
     *data = jsonData;
   }