Merge pull request #22017 from taosdata/enh/TD-25083-3.0

enh: exclude tk_log from timeseries check

source/dnode/vnode/inc/vnode.h

@@ -78,6 +78,7 @@ ESyncRole vnodeGetRole(SVnode *pVnode);
 int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
 int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
 int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void* arg1), void *arg);
 void   *vnodeGetIdx(void *pVnode);
 void   *vnodeGetIvtIdx(void *pVnode);
 
@@ -126,6 +127,9 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
 int32_t  metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
 int32_t  metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
                                int32_t payloadLen);
+bool     metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
+int32_t  metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
+int32_t  metaSizeOfTbFilterCache(void *pVnode, int8_t type);
 
 int32_t  metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);
 

source/dnode/vnode/src/meta/metaCache.c

@@ -66,6 +66,10 @@ struct SMetaCache {
     SHashObj*     pTableEntry;
     SLRUCache*    pResCache;
   } STbGroupResCache;
+
+  struct STbFilterCache {
+    SHashObj* pStb;
+  } STbFilterCache;
 };
 
 static void entryCacheClose(SMeta* pMeta) {
@@ -168,6 +172,12 @@ int32_t metaCacheOpen(SMeta* pMeta) {
   taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
   taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL);
 
+  pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (pCache->STbFilterCache.pStb == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err2;
+  }
+
   pMeta->pCache = pCache;
   return code;
 
@@ -193,6 +203,8 @@ void metaCacheClose(SMeta* pMeta) {
     taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock);
     taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry);
 
+    taosHashCleanup(pMeta->pCache->STbFilterCache.pStb);
+
     taosMemoryFree(pMeta->pCache);
     pMeta->pCache = NULL;
   }
@@ -880,3 +892,31 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
   metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid);
   return TSDB_CODE_SUCCESS;
 }
+
+bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
+  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+
+  if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) {
+    return true;
+  }
+
+  return false;
+}
+
+int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
+  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+
+  if (type == 0) {
+    return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0);
+  }
+
+  return 0;
+}
+
+int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
+  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+  if (type == 0) {
+    return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
+  }
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -496,6 +496,30 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
+                                  void *arg) {
+  SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
+  if (!pCur) {
+    return TSDB_CODE_FAILED;
+  }
+
+  while (1) {
+    tb_uid_t id = metaStbCursorNext(pCur);
+    if (id == 0) {
+      break;
+    }
+
+    if ((*filter) && (*filter)(arg, &id)) {
+      continue;
+    }
+
+    taosArrayPush(list, &id);
+  }
+
+  metaCloseStbCursor(pCur);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
   SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0);
   if (!pCur) {
@@ -531,6 +555,58 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
   return TSDB_CODE_SUCCESS;
 }
 
+#ifdef TD_ENTERPRISE
+#define TK_LOG_STB_NUM 19
+static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
+                                               "data_dir",
+                                               "dnodes_info",
+                                               "d_info",
+                                               "grants_info",
+                                               "keeper_monitor",
+                                               "logs",
+                                               "log_dir",
+                                               "log_summary",
+                                               "m_info",
+                                               "taosadapter_restful_http_request_fail",
+                                               "taosadapter_restful_http_request_in_flight",
+                                               "taosadapter_restful_http_request_summary_milliseconds",
+                                               "taosadapter_restful_http_request_total",
+                                               "taosadapter_system_cpu_percent",
+                                               "taosadapter_system_mem_percent",
+                                               "temp_dir",
+                                               "vgroups_info",
+                                               "vnodes_role"};
+
+// exclude stbs of taoskeeper log
+static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
+  char *dbName = strchr(pVnode->config.dbname, '.');
+  if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
+    return 0;
+  }
+  int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);
+  if (tbSize < TK_LOG_STB_NUM) {
+    for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) {
+      tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]);
+      if (suid != 0) {
+        metaPutTbToFilterCache(pVnode, suid, 0);
+      }
+    }
+    tbSize = metaSizeOfTbFilterCache(pVnode, 0);
+  }
+
+  return tbSize;
+}
+#endif
+
+static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) {
+  SVnode *pVnode = (SVnode *)arg1;
+
+  if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) {
+    return true;
+  }
+  return false;
+}
+
 int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
   SArray *suidList = NULL;
 
@@ -539,7 +615,13 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
     return TSDB_CODE_FAILED;
   }
 
-  if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) {
+  int32_t tbFilterSize = 0;
+  #ifdef TD_ENTERPRISE
+  tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode);
+  #endif
+
+  if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) ||
+      (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) {
     qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr());
     taosArrayDestroy(suidList);
     return TSDB_CODE_FAILED;