opt tag filter

source/dnode/vnode/src/meta/metaQuery.c

@@ -1261,7 +1261,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
     goto END;
   }
 
-  bool    first = true;
+  int     count = 0;
   int32_t valid = 0;
   while (1) {
     void   *entryKey = NULL, *entryVal = NULL;
@@ -1269,24 +1269,26 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
 
     valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal);
     if (valid < 0) {
-      tdbFree(entryVal);
       break;
     }
     STagIdxKey *p = entryKey;
     if (p == NULL) break;
-    if (p->type != pCursor->type) {
-      if (first) {
-        valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur);
-        if (valid < 0) break;
-        continue;
-      } else {
+
+    if (count >= 2) {
+      if (p->type != pCursor->type || p->suid != pCursor->suid) {
         break;
       }
+    } else {
+      count++;
+      if (p->type != pCursor->type) {
+        valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur);
+        if (valid < 0) {
+          break;
+        } else {
+          continue;
+        }
+      }
     }
-    if (p->suid != pKey->suid) {
-      break;
-    }
-    first = false;
     int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type);
     if (cmp == 0) {
       // match
@@ -1298,11 +1300,13 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
       }
       taosArrayPush(pUids, &tuid);
     } else if (cmp == 1) {
+      if (count >= 2) break;
       // not match but should continue to iter
     } else {
       // not match and no more result
-      break;
+      if (count >= 2) break;
     }
+    count++;
     valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur);
     if (valid < 0) {
       break;
@@ -1335,7 +1339,7 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi
   return ret;
 }
 int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags) {
-  const int32_t LIMIT = 128;
+  const int32_t LIMIT = 4096;
 
   int32_t isLock = false;
   int32_t sz = uidList ? taosArrayGetSize(uidList) : 0;
@@ -1377,7 +1381,6 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj
       int64_t *uid = taosArrayGet(uidList, i);
       taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
     }
-    return 0;
   }
 
   while (1) {

source/libs/executor/src/executil.c

@@ -43,7 +43,14 @@ typedef struct tagFilterAssist {
   SArray*   cInfoList;
 } tagFilterAssist;
 
-static int32_t removeInvalidTable(SArray* uids, SHashObj* tags);
+typedef enum {
+  FILTER_NO_LOGIC = 1,
+  FILTER_AND,
+  FILTER_OTHER,
+} FilterCondType;
+
+static FilterCondType checkTagCond(SNode* cond);
+static int32_t        removeInvalidTable(SArray* uids, SHashObj* tags);
 static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SHashObj* tags);
 static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond);
 static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
@@ -392,7 +399,8 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara
   return TSDB_CODE_SUCCESS;
 }
 
-static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* uidList, SNode* pTagCond) {
+static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* uidList, SNode* pTagCond,
+                                         SIdxFltStatus status) {
   int32_t      code = TSDB_CODE_SUCCESS;
   SArray*      pBlockList = NULL;
   SSDataBlock* pResBlock = NULL;
@@ -430,14 +438,22 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
   //  int64_t stt = taosGetTimestampUs();
   tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
 
+  FilterCondType condType = checkTagCond(pTagCond);
+
   int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond, tags);
   if (filter == -1) {
-    code = metaGetTableTags(metaHandle, suid, uidList, tags);
+    if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
+      code = metaGetTableTagsByUids(metaHandle, suid, uidList, tags);
+    } else {
+      code = metaGetTableTags(metaHandle, suid, uidList, tags);
+    }
     if (code != TSDB_CODE_SUCCESS) {
       qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid);
       terrno = code;
       goto end;
     }
+  } else {
+    qDebug("succ to get table tags from meta by tbname in cond, suid:%" PRIu64, suid);
   }
   if (suid != 0) {
     removeInvalidTable(uidList, tags);
@@ -842,6 +858,15 @@ static int tableUidCompare(const void* a, const void* b) {
   return u1 < u2 ? -1 : 1;
 }
 
+static FilterCondType checkTagCond(SNode* cond) {
+  if (nodeType(cond) == QUERY_NODE_OPERATOR) {
+    return FILTER_NO_LOGIC;
+  }
+  if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) {
+    return FILTER_AND;
+  }
+  return FILTER_OTHER;
+}
 static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) {
   int32_t ret = -1;
   if (nodeType(cond) == QUERY_NODE_OPERATOR) {
@@ -983,13 +1008,14 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
   taosMemoryFree(payload);
 }
 
-static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle) {
+static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle,
+                                 SIdxFltStatus status) {
   if (pTagCond == NULL) {
     return TSDB_CODE_SUCCESS;
   }
 
   terrno = TDB_CODE_SUCCESS;
-  SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
+  SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond, status);
   if (terrno != TDB_CODE_SUCCESS) {
     colDataDestroy(pColInfoData);
     taosMemoryFreeClear(pColInfoData);
@@ -1034,12 +1060,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
   pListInfo->suid = pScanNode->suid;
   SArray* res = taosArrayInit(8, sizeof(uint64_t));
 
+  SIdxFltStatus status = SFLT_NOT_INDEX;
   if (pScanNode->tableType != TSDB_SUPER_TABLE) {
     if (metaIsTableExist(metaHandle, tableUid)) {
       taosArrayPush(res, &tableUid);
     }
 
-    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle, status);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -1064,7 +1091,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
         SIndexMetaArg metaArg = {
             .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
 
-        SIdxFltStatus status = SFLT_NOT_INDEX;
         code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
         if (code != 0 || status == SFLT_NOT_INDEX) {
           qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
@@ -1073,7 +1099,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
       }
     }
 
-    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle, status);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }