TD-6129<feature> add tag-> where logic

src/client/CMakeLists.txt

@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(inc)
 INCLUDE_DIRECTORIES(jni)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc)

src/client/inc/tscUtil.h

@@ -29,6 +29,7 @@ extern "C" {
 #include "tsched.h"
 #include "tsclient.h"
 #include "tglobal.h"
+#include "tsdbMeta.h"
 
 #define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
@@ -379,6 +380,9 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
 char* parseTagDatatoJson(void *p);
 void findTagValue(void* data, char* key, int32_t keyLen, char* out, int16_t len);
 
+int8_t  jsonType2DbType(double data, int jsonType);
+void*      getJsonTagValue(STable* pTable, char* key);
+
 #ifdef __cplusplus
 }
 #endif

src/client/src/tscSQLParser.c

@@ -5110,7 +5110,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));
     ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
     //if (ret == TSDB_CODE_SUCCESS) {
-    //  ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList));
+    //  ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList), NULL);
     //}
     
     SBufferWriter bw = tbufInitWriter(NULL, false);

src/client/src/tscUtil.c

@@ -73,7 +73,6 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
 
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-    case TSDB_DATA_TYPE_JSON:
       if (bufSize < 0) {
         tscError("invalid buf size");
         return TSDB_CODE_TSC_INVALID_VALUE;
@@ -5271,10 +5270,11 @@ char* parseTagDatatoJson(void *p){
     }else{  // json value
       char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
       char* realData = POINTER_SHIFT(val, CHAR_BYTES);
-      if(*(char*)val == cJSON_String){
-        if (JSON_TYPE_BINARY){
+      char type = *(char*)val;
+      if(IS_VAR_DATA_TYPE(type)){
+        if (type == TSDB_DATA_TYPE_BINARY){
           strncpy(tagJsonValue, varDataVal(realData), varDataLen(realData));
-        } else if(JSON_TYPE_NCHAR) {
+        } else if(type == TSDB_DATA_TYPE_NCHAR) {
           int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue);
           if (length == 0) {
             tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
@@ -5288,7 +5288,7 @@ char* parseTagDatatoJson(void *p){
           goto end;
         }
         cJSON_AddItemToObject(json, tagJsonKey, value);
-      }else if(*(char*)val == cJSON_Number){
+      }else if(type == TSDB_DATA_TYPE_DOUBLE){
         double jsonVd = *(double*)(realData);
         cJSON* value = cJSON_CreateNumber(jsonVd);
         if (value == NULL)
@@ -5296,7 +5296,16 @@ char* parseTagDatatoJson(void *p){
           goto end;
         }
         cJSON_AddItemToObject(json, tagJsonKey, value);
-      }else{
+      }else if(type == TSDB_DATA_TYPE_BIGINT){
+        int64_t jsonVd = *(int64_t*)(realData);
+        cJSON* value = cJSON_CreateNumber(jsonVd);
+        if (value == NULL)
+        {
+          goto end;
+        }
+        cJSON_AddItemToObject(json, tagJsonKey, value);
+      }
+      else{
         tscError("unsupportted json value");
       }
     }
@@ -5355,7 +5364,7 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
     memset(tagVal, 0, TSDB_MAX_TAGS_LEN);
     if(item->type == cJSON_String){     // add json value  format: type|data
       outLen = 0;
-      *tagVal = item->type;     // type
+      *tagVal = jsonType2DbType(0, item->type);     // type
       char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
       if (JSON_TYPE_BINARY){
         strncpy(tagVal, item->valuestring, strlen(item->valuestring));
@@ -5372,7 +5381,7 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       varDataSetLen(tagData, outLen);
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, true);
     }else if(item->type == cJSON_Number){
-      *tagVal = item->type;    // type
+      *tagVal = jsonType2DbType(item->valuedouble, item->type);    // type
       char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
       *((double *)tagData) = item->valuedouble;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_BIGINT, tagVal, true);
@@ -5386,3 +5395,46 @@ end:
   cJSON_Delete(root);
   return retCode;
 }
+
+int8_t jsonType2DbType(double data, int jsonType){
+  switch(jsonType){
+    case cJSON_Number:
+      if (data - (int64_t)data > 0) return TSDB_DATA_TYPE_DOUBLE; else return TSDB_DATA_TYPE_BIGINT;
+    case cJSON_String:
+      if (JSON_TYPE_NCHAR) return TSDB_DATA_TYPE_NCHAR; else return TSDB_DATA_TYPE_BINARY;
+  }
+  return TSDB_DATA_TYPE_NULL;
+}
+
+void* getJsonTagValue(STable* pTable, char* key){
+  int32_t outLen = 0;
+  if(JSON_TYPE_NCHAR){
+    char tagKey[256] = {0};
+    if (!taosMbsToUcs4(key, strlen(key), tagKey, 256, &outLen)) {
+      tscError("json key to ucs4 error:%s|%s", strerror(errno), key);
+      return NULL;
+    }
+    key = tagKey;
+  }else{
+    outLen = strlen(key);
+  }
+  if(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE){
+    STable* superTable= pTable->pSuper;
+    SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen);
+    if(data == NULL) return NULL;
+    JsonMapValue jmvalue = {pTable, 0};
+    JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ);
+    if (p == NULL) return NULL;
+    int16_t valId = p->colId + 1;
+    return POINTER_SHIFT(kvRowValues(pTable->tagVal), valId);
+  }else if(TABLE_TYPE(pTable) == TSDB_SUPER_TABLE){
+    SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen);
+    if(data == NULL) return NULL;
+    if(taosArrayGetSize(*data) == 0) return NULL;
+    JsonMapValue* p = taosArrayGet(*data, 0);
+    int16_t valId = p->colId + 1;
+    return POINTER_SHIFT(kvRowValues(((STable*)p->table)->tagVal), valId);
+  }
+  return NULL;
+}
+

src/common/inc/tvariant.h

@@ -39,6 +39,8 @@ typedef struct tVariant {
 
 bool tVariantIsValid(tVariant *pVar);
 
+bool tVariantTypeMatch(tVariant *pVar, int8_t dbType);
+
 void tVariantCreate(tVariant *pVar, SStrToken *token);
 
 void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type);

src/common/src/tvariant.c

@@ -22,7 +22,6 @@
 #include "ttype.h"
 #include "tutil.h"
 #include "tvariant.h"
-#include "tscUtil.h"
 
 #define SET_EXT_INFO(converted, res, minv, maxv, exti) do {                                                       \
                                                         if (converted == NULL || exti == NULL || *converted == false) { break; }  \
@@ -212,6 +211,36 @@ bool tVariantIsValid(tVariant *pVar) {
   return isValidDataType(pVar->nType);
 }
 
+bool tVariantTypeMatch(tVariant *pVar, int8_t dbType){
+  switch (dbType) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR: {
+      if(pVar->nType != TSDB_DATA_TYPE_BINARY && pVar->nType != TSDB_DATA_TYPE_NCHAR){
+        return false;
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE:{
+      if(pVar->nType == TSDB_DATA_TYPE_BINARY && pVar->nType == TSDB_DATA_TYPE_NCHAR){
+        return false;
+      }
+      break;
+    }
+  }
+  return true;
+}
+
 void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
   

src/inc/taoserror.h

@@ -288,7 +288,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistency in replica")
 #define TSDB_CODE_QRY_INVALID_TIME_CONDITION    TAOS_DEF_ERROR_CODE(0, 0x070D)  //"invalid time condition")
 #define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070E)  //"System error")
-
+#define TSDB_CODE_QRY_JSON_KEY_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x070F)  //"json tag key not exist")
+#define TSDB_CODE_QRY_JSON_KEY_TYPE_ERROR       TAOS_DEF_ERROR_CODE(0, 0x0710)  //"json tag key type not match")
 
 // grant
 #define TSDB_CODE_GRANT_EXPIRED                 TAOS_DEF_ERROR_CODE(0, 0x0800)  //"License expired")

src/inc/tsdb.h

@@ -418,6 +418,7 @@ int tsdbCompact(STsdbRepo *pRepo);
 // no problem return true
 bool tsdbNoProblem(STsdbRepo* pRepo);
 
+
 #ifdef __cplusplus
 }
 #endif

src/query/inc/qFilter.h

@@ -245,7 +245,8 @@ typedef struct SFilterInfo {
   uint16_t          blkGroupNum;
   uint16_t         *blkUnits;
   int8_t           *blkUnitRes;
-  
+  void             *pTable;
+
   SFilterPCtx       pctx;
 } SFilterInfo;
 

src/query/inc/qUtil.h

@@ -109,5 +109,4 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset);
 
 int32_t initUdfInfo(SUdfInfo* pUdfInfo);
-
 #endif  // TDENGINE_QUERYUTIL_H

src/query/src/qFilter.c

@@ -18,6 +18,7 @@
 #include "tcompare.h"
 #include "hash.h"
 #include "tscUtil.h"
+#include "tsdbMeta.h"
 
 OptrStr gOptrStr[] = {
   {TSDB_RELATION_INVALID,                  "invalid"},
@@ -3174,21 +3175,31 @@ int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_
 }
 
 // convert json type for next compare and so on
-void filterJsonTypeConvert(SFilterInfo* info) {
-  uint8_t type = 0;
-  if(JSON_TYPE_NCHAR){ type = TSDB_DATA_TYPE_NCHAR;} else {type = TSDB_DATA_TYPE_BINARY;}
-  for(int i = 0; i < info->unitNum; i++){
-    if(info->units[i].compare.type == TSDB_DATA_TYPE_JSON){
-      info->units[i].compare.type= type;
-    }
-  }
-
+int filterJsonTypeConvert(SFilterInfo* info) {
   for(int i = 0; i < info->fields[FLD_TYPE_COLUMN].num; i++) {
     SSchema* schema = info->fields[FLD_TYPE_COLUMN].fields[i].desc;
     if(schema->type == TSDB_DATA_TYPE_JSON){
+
+      void* data = getJsonTagValue(info->pTable, schema->name);
+      if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
+      int8_t type = *(char*)data;
+      assert(type > TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_JSON);
       schema->type = type;
     }
   }
+  for(int i = 0; i < info->unitNum; i++){
+    if(info->units[i].compare.type == TSDB_DATA_TYPE_JSON){
+      SFilterField *colLeft = FILTER_UNIT_LEFT_FIELD(info, &info->units[i]);
+      info->units[i].compare.type = FILTER_GET_COL_FIELD_TYPE(colLeft);
+
+      SFilterField *colRight = FILTER_UNIT_RIGHT_FIELD(info, &info->units[i]);
+      tVariant* var = colRight->desc;
+      if(!tVariantTypeMatch(var, info->units[i].compare.type))
+        return TSDB_CODE_QRY_JSON_KEY_TYPE_ERROR;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) {
@@ -3211,7 +3222,11 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) {
 
   code = filterTreeToGroup(tree, info, group);
   ERR_JRET(code);
-  filterJsonTypeConvert(info);
+
+  if(info->pTable){
+    code = filterJsonTypeConvert(info);
+    ERR_JRET(code);
+  }
 
   filterConvertGroupFromArray(info, group);
 

src/query/src/qUtil.c

@@ -23,8 +23,8 @@
 #include "tlosertree.h"
 #include "queryLog.h"
 #include "tscompression.h"
-#include "tsdbMeta.h"
 #include "tscUtil.h"
+#include "cJSON.h"
 
 typedef struct SCompSupporter {
   STableQueryInfo **pTableQueryInfo;

src/tsdb/CMakeLists.txt

@@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tsdb ${SRC})

src/tsdb/inc/tsdbMeta.h

@@ -97,7 +97,7 @@ int16_t    tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId);
 int        tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema);
 STSchema*  tsdbGetTableLatestSchema(STable *pTable);
 void       tsdbFreeLastColumns(STable* pTable);
-int        tscCompareJsonMapValue(const void* a, const void* b);
+int        tsdbCompareJsonMapValue(const void* a, const void* b);
 
 static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
   if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {

src/tsdb/src/tsdbMeta.c

@@ -1141,9 +1141,9 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
         }
       }
       JsonMapValue jmvalue = {pTable, pColIdx->colId};
-      void* p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_EQ);
+      void* p = taosArraySearch(tablistNew, &jmvalue, tsdbCompareJsonMapValue, TD_EQ);
       if (p == NULL) {
-        p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_GE);
+        p = taosArraySearch(tablistNew, &jmvalue, tsdbCompareJsonMapValue, TD_GE);
         if(p == NULL){
           taosArrayPush(tablistNew, &jmvalue);
         }else{
@@ -1188,7 +1188,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
       }
 
       JsonMapValue jmvalue = {pTable, pColIdx->colId};
-      void* p = taosArraySearch(*tablist, &jmvalue, tscCompareJsonMapValue, TD_EQ);
+      void* p = taosArraySearch(*tablist, &jmvalue, tsdbCompareJsonMapValue, TD_EQ);
       if (p == NULL) {
         tsdbError("json tag no tableid error,%d", j);
         continue;

src/tsdb/src/tsdbRead.c

@@ -26,6 +26,7 @@
 #include "tsdbint.h"
 #include "texpr.h"
 #include "qFilter.h"
+#include "tscUtil.h"
 
 #define EXTRA_BYTES 2
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)
@@ -2685,7 +2686,7 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
       SArray* tallistOld = *pRecord;
       for (int i = 0; i < taosArrayGetSize(tallistOld); ++i) {
         void* p = taosArrayGet(tallistOld, i);
-        void* pFind = taosArraySearch(tablist, p, tscCompareJsonMapValue, TD_EQ);
+        void* pFind = taosArraySearch(tablist, p, tsdbCompareJsonMapValue, TD_EQ);
         if(pFind == NULL){
           taosArrayPush(tablist, p);
         }
@@ -3774,9 +3775,11 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
     // TODO: more error handling
   } END_TRY
 
-  void *filterInfo = NULL;
+  void *filterInfo = calloc(1, sizeof(SFilterInfo));
+  ((SFilterInfo*)filterInfo)->pTable = pTable;
   ret = filterInitFromTree(expr, &filterInfo, 0);
   if (ret != TSDB_CODE_SUCCESS) {
+    filterFreeInfo(filterInfo);
     terrno = ret;
     goto _error;
   }
@@ -3799,6 +3802,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons
   return ret;
 
   _error:
+
   taosArrayDestroy(res);
   return terrno;
 }
@@ -4064,30 +4068,6 @@ static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray*
   tSkipListDestroyIter(iter);
 }
 
-void* getJsonTagValue(STable* pTable, char* key){
-  assert(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE);
-  int32_t outLen = 0;
-  if(JSON_TYPE_NCHAR){
-    char tagKey[256] = {0};
-    if (!taosMbsToUcs4(key, strlen(key), tagKey, 256, &outLen)) {
-      tsdbError("json key to ucs4 error:%s|%s", strerror(errno), key);
-      return NULL;
-    }
-    key = tagKey;
-  }else{
-    outLen = strlen(key);
-  }
-  STable* superTable = pTable->pSuper;
-  SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen);
-  if(data == NULL) return NULL;
-  JsonMapValue jmvalue = {pTable, 0};
-  JsonMapValue* p = taosArraySearch(*data, &jmvalue, tscCompareJsonMapValue, TD_EQ);
-  if (p == NULL) return NULL;
-  int16_t valId = p->colId + 1;
-  return POINTER_SHIFT(kvRowValues(pTable->tagVal), valId);
-}
-
-
 static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, char* name, void **data) {
   JsonMapValue* jsonMapV = (JsonMapValue*)(param);
   STable* pTable = (STable*)(jsonMapV->table);
@@ -4095,7 +4075,9 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch
   if (id == TSDB_TBNAME_COLUMN_INDEX) {
     *data = TABLE_NAME(pTable);
   } else {
-    *data = getJsonTagValue(pTable, name);
+    void* jsonData = getJsonTagValue(pTable, name);
+    if (jsonData != NULL) jsonData += CHAR_BYTES;   // jump type
+    *data = jsonData;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -4129,9 +4111,9 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){
     }else{
       for(int j = 0; j < taosArrayGetSize(*data); j++){
         void* element = taosArrayGet(*data, j);
-        void* p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_EQ);
+        void* p = taosArraySearch(tabList, element, tsdbCompareJsonMapValue, TD_EQ);
         if (p == NULL) {
-          p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_GE);
+          p = taosArraySearch(tabList, element, tsdbCompareJsonMapValue, TD_GE);
           if(p == NULL){
             taosArrayPush(tabList, tabList);
           }else{