TD-6129<feature> add tag-> where logic

src/inc/taoserror.h

@@ -269,6 +269,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_MESSED_MSG                TAOS_DEF_ERROR_CODE(0, 0x0614)  //"TSDB messed message")
 #define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0615)  //"TSDB invalid tag value")
 #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0616)  //"TSDB no cache last row data")
+#define TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE      TAOS_DEF_ERROR_CODE(0, 0x0617)  //"TSDB invalid same json value")
+#define TSDB_CODE_TDB_NO_JSON_TAG_KEY           TAOS_DEF_ERROR_CODE(0, 0x0618)  //"TSDB no tag json key")
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)  //"Invalid handle")

src/query/inc/qFilter.h

@@ -113,6 +113,7 @@ typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const v
 typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
 typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t);
 typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
 
 typedef struct SFilterRangeCompare {
   int64_t s;
@@ -333,6 +334,7 @@ typedef struct SFilterInfo {
 extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
 extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols);
 extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
+extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
 extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
 extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
 extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);

src/query/src/qFilter.c

@@ -1542,7 +1542,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
           SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit);
           char *data = right->data;
-          if (IS_VAR_DATA_TYPE(type) {
+          if (IS_VAR_DATA_TYPE(type)) {
             tlen = varDataLen(data);
             data += VARSTR_HEADER_SIZE;
           }
@@ -3152,6 +3152,43 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) {
+  CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
+  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
+
+  if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+    SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
+    SSchema* sch = fi->desc;
+
+    (*fp)(param, sch->colId, sch->name, &fi->data);
+  }
+
+  filterUpdateComUnits(info);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+// convert json type for next compare and so on
+void filterJsonTypeConvert(SFilterInfo* info) {
+  uint8_t type = 0;
+  if(JSON_TYPE_NCHAR){ type = TSDB_DATA_TYPE_NCHAR;} else {type = TSDB_DATA_TYPE_BINARY;}
+  for(int i = 0; i < info->unitNum; i++){
+    if(info->units[info->unitNum].compare.type == TSDB_DATA_TYPE_JSON){
+      info->units[info->unitNum].compare.type= type;
+    }
+  }
+
+  for(int i = 0; i < info->fields[FLD_TYPE_COLUMN].num; i++) {
+    SSchema* schema = info->fields[FLD_TYPE_COLUMN].fields[i].desc;
+    if(schema->type == TSDB_DATA_TYPE_JSON){
+      schema->type = type;
+    }
+  }
+}
 
 int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -3172,8 +3209,8 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) {
   filterInitUnitsFields(info);
 
   code = filterTreeToGroup(tree, info, group);
-
   ERR_JRET(code);
+  filterJsonTypeConvert(info);
 
   filterConvertGroupFromArray(info, group);
 

src/query/src/qUtil.c

@@ -23,6 +23,8 @@
 #include "tlosertree.h"
 #include "queryLog.h"
 #include "tscompression.h"
+#include "tsdbMeta.h"
+#include "tscUtil.h"
 
 typedef struct SCompSupporter {
   STableQueryInfo **pTableQueryInfo;
@@ -725,4 +727,3 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi
     tfree(outputBuf);
   }
 }
-

src/tsdb/src/tsdbMeta.c

@@ -1127,12 +1127,30 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
       }else{
         tablistNew = *tablist;
       }
+
+      if(taosArrayGetSize(tablistNew) > 0){
+        // validate type
+        JsonMapValue* tmp = taosArrayGet(tablistNew, 0);
+        void* data1 = tdGetKVRowValOfCol(((STable *)(tmp->table))->tagVal, tmp->colId + 1);
+        SColIdx * pInsertColIdx = kvRowColIdxAt(pTable->tagVal, j + 1);
+        void* data2 = (kvRowColVal(pTable->tagVal, pInsertColIdx));
+        if(*(char*)data1 != *(char*)data2){
+          terrno = TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE;
+          tsdbError("invalidate same json tag value");
+          return -1;
+        }
+      }
       JsonMapValue jmvalue = {pTable, pColIdx->colId};
       void* p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_EQ);
       if (p == NULL) {
-        taosArrayPush(tablistNew, &jmvalue);
+        p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_GE);
+        if(p == NULL){
+          taosArrayPush(tablistNew, &jmvalue);
+        }else{
+          taosArrayInsert(tablistNew, TARRAY_ELEM_IDX(tablistNew, p), &jmvalue);
+        }
       }else{
-        taosArrayInsert(tablistNew, TARRAY_ELEM_IDX(tablistNew, p), &jmvalue);
+        tsdbError("insert dumplicate");
       }
     }
   }else{

src/tsdb/src/tsdbRead.c

@@ -4064,33 +4064,90 @@ static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray*
   tSkipListDestroyIter(iter);
 }
 
-static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, void **data) {
+void* getJsonTagValue(STable* pTable, char* key){
+  assert(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE);
+  int32_t outLen = 0;
+  if(JSON_TYPE_NCHAR){
+    char tagKey[256] = {0};
+    if (!taosMbsToUcs4(key, strlen(key), tagKey, 256, &outLen)) {
+      tsdbError("json key to ucs4 error:%s|%s", strerror(errno), key);
+      return NULL;
+    }
+    key = tagKey;
+  }else{
+    outLen = strlen(key);
+  }
+  STable* superTable = pTable->pSuper;
+  SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen);
+  if(data == NULL) return NULL;
+  JsonMapValue jmvalue = {pTable, 0};
+  JsonMapValue* p = taosArraySearch(*data, &jmvalue, tscCompareJsonMapValue, TD_EQ);
+  if (p == NULL) return NULL;
+  int16_t valId = p->colId + 1;
+  return POINTER_SHIFT(kvRowValues(pTable->tagVal), valId);
+}
+
+
+static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, char* name, void **data) {
   JsonMapValue* jsonMapV = (JsonMapValue*)(param);
   STable* pTable = (STable*)(jsonMapV->table);
 
   if (id == TSDB_TBNAME_COLUMN_INDEX) {
     *data = TABLE_NAME(pTable);
   } else {
-    *data = tdGetKVRowValOfCol(pTable->tagVal, jsonMapV->colId + 1);
+    *data = getJsonTagValue(pTable, name);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-static void queryByJsonTag(SArray* pTableList, void* filterInfo, SArray* res){
-  int32_t size = taosArrayGetSize(pTableList);
+static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){
+  // get all table in fields, and dumplicate it
+  SArray* tabList = NULL;
+  SFilterInfo* info = (SFilterInfo*)filterInfo;
+  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+    SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
+    SSchema*      sch = fi->desc;
+    if (sch-> colId == TSDB_TBNAME_COLUMN_INDEX) continue;
+    SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, sch->name, strlen(sch->name));
+    if(data == NULL) continue;
+    if(tabList == NULL) {
+      tabList = taosArrayDup(*data);
+    }else{
+      for(int j = 0; j < taosArrayGetSize(*data); j++){
+        void* element = taosArrayGet(*data, j);
+        void* p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_EQ);
+        if (p == NULL) {
+          p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_GE);
+          if(p == NULL){
+            taosArrayPush(tabList, tabList);
+          }else{
+            taosArrayInsert(tabList, TARRAY_ELEM_IDX(tabList, p), tabList);
+          }
+        }
+      }
+    }
+  }
+  if(tabList == NULL || taosArrayGetSize(tabList) == 0){
+    tsdbError("json key not exist");
+    terrno = TSDB_CODE_TDB_NO_JSON_TAG_KEY;
+    taosArrayDestroy(tabList);
+    return;
+  }
+  int32_t size = taosArrayGetSize(tabList);
   int8_t *addToResult = NULL;
   for(int i = 0; i < size; i++){
-    JsonMapValue* data = taosArrayGet(pTableList, i);
-    filterSetColFieldData(filterInfo, data, tsdbGetJsonTagDataFromId);
+    JsonMapValue* data = taosArrayGet(tabList, i);
+    filterSetJsonColFieldData(filterInfo, data, tsdbGetJsonTagDataFromId);
     bool all = filterExecute(filterInfo, 1, &addToResult, NULL, 0);
 
     if (all || (addToResult && *addToResult)) {
-      STableKeyInfo info = {.pTable = (void*)(data->table), .lastKey = TSKEY_INITIAL_VAL};
-      taosArrayPush(res, &info);
+      STableKeyInfo kInfo = {.pTable = (void*)(data->table), .lastKey = TSKEY_INITIAL_VAL};
+      taosArrayPush(res, &kInfo);
     }
   }
   tfree(addToResult);
+  taosArrayDestroy(tabList);
 }
 
 static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo) {