Merge pull request #18468 from taosdata/feature/3_liaohj

refactor: improve the perf in query

cmake/cmake.define

@@ -123,14 +123,20 @@ ELSE ()
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
     ENDIF ()
 
-    MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
     IF (TD_INTEL_64 OR TD_INTEL_32)
         ADD_DEFINITIONS("-msse4.2")
         IF("${FMA_SUPPORT}" MATCHES "true")
-            MESSAGE(STATUS "turn fma function support on")
+            MESSAGE(STATUS "fma function supported")
             ADD_DEFINITIONS("-mfma")
         ELSE ()
-            MESSAGE(STATUS "turn fma function support off")
+            MESSAGE(STATUS "fma function NOT supported")
+        ENDIF()
+
+        IF("${SIMD_SUPPORT}" MATCHES "true")
+            ADD_DEFINITIONS("-mavx -mavx2")
+            MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED")
+        ELSE()
+            MESSAGE(STATUS "SIMD instruction (AVX/AVX2)is NOT ACTIVATED")
         ENDIF()
     ENDIF ()
 

cmake/cmake.platform

 cmake_minimum_required(VERSION 3.0)
 
-MESSAGE("Current system is ${CMAKE_SYSTEM_NAME}")
-
 # init
 SET(TD_LINUX FALSE)
 SET(TD_WINDOWS FALSE)
 SET(TD_DARWIN FALSE)
 
-MESSAGE("Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
 if(CMAKE_COMPILER_IS_GNUCXX MATCHES 1)
     set(CXX_COMPILER_IS_GNU TRUE)
 else()
     set(CXX_COMPILER_IS_GNU FALSE)
 endif()
 
-MESSAGE("Current system name is ${CMAKE_SYSTEM_NAME}.")
+MESSAGE("Current system: ${CMAKE_SYSTEM_NAME}")
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 
@@ -26,6 +23,8 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
     ENDIF ()
 
+    MESSAGE("Current system processor: ${CMAKE_SYSTEM_PROCESSOR}")
+
     IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 
         SET(TD_LINUX TRUE)
@@ -44,7 +43,6 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
         SET(OSTYPE "macOS")
         ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare")
 
-        MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
         IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
             MESSAGE("Current system arch is arm64")
             SET(TD_DARWIN_64 TRUE)
@@ -80,24 +78,22 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
 ENDIF()
 
 IF ("${CPUTYPE}" STREQUAL "")
-  MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected")
-
   IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
-    MESSAGE(STATUS "The current platform is amd64")
+    MESSAGE(STATUS "Current platform is amd64")
     SET(PLATFORM_ARCH_STR "amd64")
     SET(TD_INTEL_64 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
-    MESSAGE(STATUS "The current platform is x86")
+    MESSAGE(STATUS "Current platform is x86")
     SET(PLATFORM_ARCH_STR "i386")
     SET(TD_INTEL_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
-    MESSAGE(STATUS "The current platform is aarch32")
+    MESSAGE(STATUS "Current platform is aarch32")
     SET(PLATFORM_ARCH_STR "arm")
     SET(TD_ARM_32 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
     ADD_DEFINITIONS("-D_TD_ARM_32")
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
-    MESSAGE(STATUS "The current platform is aarch64")
+    MESSAGE(STATUS "Current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     SET(TD_ARM_64 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
@@ -151,5 +147,5 @@ ENDIF ()
 
 MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR})
 
-MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}")
-MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
+MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
+MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/cmake.version

@@ -26,7 +26,7 @@ ELSEIF (HAVE_GIT)
     SET(TD_VER_GIT "no git commit id")
   ENDIF ()
 ELSE ()
-  message(STATUS "no git cmd")
+  message(STATUS "no git found")
   SET(TD_VER_GIT "no git commit id")
 ENDIF ()
 
@@ -70,9 +70,9 @@ MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE})
 MESSAGE(STATUS "commit id:  " ${TD_VER_GIT})
 MESSAGE(STATUS "build date: " ${TD_VER_DATE})
 MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE})
-MESSAGE(STATUS "type: " ${TD_VER_VERTYPE})
-MESSAGE(STATUS "cpu:  " ${TD_VER_CPUTYPE})
-MESSAGE(STATUS "os:   " ${TD_VER_OSTYPE})
+MESSAGE(STATUS "type:       " ${TD_VER_VERTYPE})
+MESSAGE(STATUS "cpu:        " ${TD_VER_CPUTYPE})
+MESSAGE(STATUS "os:         " ${TD_VER_OSTYPE})
 MESSAGE(STATUS "============= compile version parameter information end  ============= ")
 
 STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})

include/common/tcommon.h

@@ -225,13 +225,13 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
-  char*         pData;    // the corresponding block data in memory
+  char*         pData;       // the corresponding block data in memory
   union {
     char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
-  SColumnInfo info;     // column info
-  bool        hasNull;  // if current column data has null value.
+  SColumnInfo   info;        // column info
+  bool          hasNull;     // if current column data has null value.
 } SColumnInfoData;
 
 typedef struct SQueryTableDataCond {

include/common/tdatablock.h

@@ -24,11 +24,6 @@
 extern "C" {
 #endif
 
-typedef struct SCorEpSet {
-  int32_t version;
-  SEpSet  epSet;
-} SCorEpSet;
-
 typedef struct SBlockOrderInfo {
   bool             nullFirst;
   int32_t          order;
@@ -36,14 +31,6 @@ typedef struct SBlockOrderInfo {
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
-int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
-void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
-
-bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
-
-void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
-SEpSet getEpSet_s(SCorEpSet* pEpSet);
-
 #define NBIT                     (3u)
 #define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
 #define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])

include/common/tglobal.h

@@ -44,6 +44,8 @@ extern int32_t tsCompatibleModel;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerMin[3];
 extern int32_t tsCountAlwaysReturnValue;
+extern float   tsSelectivityRatio;
+extern int32_t tsTagFilterResCacheSize;
 
 // queue & threads
 extern int32_t tsNumOfRpcThreads;

include/common/tmisce.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TMISCE_H
+#define TDENGINE_TMISCE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tmsg.h"
+
+typedef struct SCorEpSet {
+  int32_t version;
+  SEpSet  epSet;
+} SCorEpSet;
+
+int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
+void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
+bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
+
+void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
+SEpSet getEpSet_s(SCorEpSet* pEpSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TMISCE_H

include/common/tmsg.h

@@ -649,34 +649,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR
 int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
 void    tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
 
-typedef struct {
-  int16_t lowerRelOptr;
-  int16_t upperRelOptr;
-  int16_t filterstr;  // denote if current column is char(binary/nchar)
-
-  union {
-    struct {
-      int64_t lowerBndi;
-      int64_t upperBndi;
-    };
-    struct {
-      double lowerBndd;
-      double upperBndd;
-    };
-    struct {
-      int64_t pz;
-      int64_t len;
-    };
-  };
-} SColumnFilterInfo;
-
-typedef struct {
-  int16_t numOfFilters;
-  union {
-    int64_t            placeholder;
-    SColumnFilterInfo* filterInfo;
-  };
-} SColumnFilterList;
 /*
  * for client side struct, only column id, type, bytes are necessary
  * But for data in vnode side, we need all the following information.
@@ -687,10 +659,10 @@ typedef struct {
     int16_t  slotId;
   };
 
-  int8_t  type;
-  int32_t bytes;
   uint8_t precision;
   uint8_t scale;
+  int32_t bytes;
+  int8_t  type;
 } SColumnInfo;
 
 typedef struct STimeWindow {

include/libs/executor/executor.h

@@ -142,8 +142,11 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
  */
 
 int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal);
+
 int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds);
 
+void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
+
 /**
  * kill the ongoing query asynchronously
  * @param tinfo  qhandle

include/libs/function/function.h

@@ -115,7 +115,7 @@ typedef struct SInputColumnInfoData {
   int32_t           startRowIndex;    // handle started row index
   int32_t           numOfRows;        // the number of rows needs to be handled
   int32_t           numOfInputCols;   // PTS is not included
-  bool              colDataAggIsSet;  // if agg is set or not
+  bool              colDataSMAIsSet;  // if agg is set or not
   SColumnInfoData  *pPTS;             // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;

include/os/os.h

@@ -43,6 +43,7 @@ extern "C" {
 #include <sys/utsname.h>
 #include <sys/wait.h>
 #include <termios.h>
+#include <cpuid.h>
 
 #if defined(DARWIN)
 #else
@@ -82,6 +83,12 @@ extern "C" {
 #include <wchar.h>
 #include <wctype.h>
 
+#if __AVX__
+#include <immintrin.h>
+#elif __SSE4_2__
+#include <nmmintrin.h>
+#endif
+
 #include "osThread.h"
 
 #include "osAtomic.h"

include/os/osDef.h

@@ -168,22 +168,22 @@ void syslog(int unused, const char *format, ...);
     }                            \
   } while (0)
 
-#define DEFAULT_DOUBLE_COMP(x, y) \
-  do {                            \
-    if (isnan(x) && isnan(y)) {   \
-      return 0;                   \
-    }                             \
-    if (isnan(x)) {               \
-      return -1;                  \
-    }                             \
-    if (isnan(y)) {               \
-      return 1;                   \
-    }                             \
-    if ((x) == (y)) {             \
-      return 0;                   \
-    } else {                      \
-      return (x) < (y) ? -1 : 1;  \
-    }                             \
+#define DEFAULT_DOUBLE_COMP(x, y)         \
+  do {                                    \
+    if (isnan(x) && isnan(y)) {           \
+      return 0;                           \
+    }                                     \
+    if (isnan(x)) {                       \
+      return -1;                          \
+    }                                     \
+    if (isnan(y)) {                       \
+      return 1;                           \
+    }                                     \
+    if (fabs((x) - (y)) <= DBL_EPSILON) { \
+      return 0;                           \
+    } else {                              \
+      return (x) < (y) ? -1 : 1;          \
+    }                                     \
   } while (0)
 
 #define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y)

include/os/osEnv.h

@@ -36,6 +36,11 @@ extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
 extern char           *tsProcPath;
+extern char            tsSIMDEnable;
+extern char            tsSSE42Enable;
+extern char            tsAVXEnable;
+extern char            tsAVX2Enable;
+extern char            tsFMAEnable;
 
 extern char configDir[];
 extern char tsDataDir[];

include/os/osMemory.h

@@ -37,6 +37,7 @@ void    taosMemoryFree(void *ptr);
 int64_t taosMemorySize(void *ptr);
 void    taosPrintBackTrace();
 void    taosMemoryTrim(int32_t size);
+void   *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
 
 #define taosMemoryFreeClear(ptr)   \
   do {                             \

include/os/osSysinfo.h

@@ -40,6 +40,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
 int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
 int32_t taosGetCpuCores(float *numOfCores);
 void    taosGetCpuUsage(double *cpu_system, double *cpu_engine);
+int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma);
 int32_t taosGetTotalMemory(int64_t *totalKB);
 int32_t taosGetProcMemory(int64_t *usedKB);
 int32_t taosGetSysMemory(int64_t *usedKB);

include/util/tarray.h

@@ -104,6 +104,15 @@ static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) {
   return taosArrayAddBatch(pArray, pData, 1);
 }
 
+/**
+ * @brief reserve the capacity of the array
+ *
+ * @param pArray
+ * @param num
+ * @return void* the start position of the reserved memory
+ */
+void* taosArrayReserve(SArray* pArray, int32_t num);
+
 /**
  *
  * @param pArray

include/util/tlist.h

@@ -225,7 +225,7 @@ void      *tdListFree(SList *list);
 void       tdListPrependNode(SList *list, SListNode *node);
 void       tdListAppendNode(SList *list, SListNode *node);
 int32_t    tdListPrepend(SList *list, void *data);
-int32_t    tdListAppend(SList *list, void *data);
+int32_t    tdListAppend(SList *list, const void *data);
 SListNode *tdListPopHead(SList *list);
 SListNode *tdListPopTail(SList *list);
 SListNode *tdListGetHead(SList *list);

include/util/tpagedbuf.h

@@ -24,12 +24,9 @@
 extern "C" {
 #endif
 
-typedef struct SArray*       SIDList;
 typedef struct SPageInfo     SPageInfo;
 typedef struct SDiskbasedBuf SDiskbasedBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024LL)  // in bytes
-
 typedef struct SFilePage {
   int32_t num;
   char    data[];
@@ -69,7 +66,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId);
  * @param pBuf
  * @return
  */
-SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf);
+SArray* getDataBufPagesIdList(SDiskbasedBuf* pBuf);
 
 /**
  * get the specified buffer page by id
@@ -111,7 +108,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf);
  * @param pList
  * @return
  */
-SPageInfo* getLastPageInfo(SIDList pList);
+SPageInfo* getLastPageInfo(SArray* pList);
 
 /**
  *

source/client/inc/clientInt.h

@@ -26,7 +26,7 @@ extern "C" {
 #include "query.h"
 #include "taos.h"
 #include "tcommon.h"
-#include "tdatablock.h"
+#include "tmisce.h"
 #include "tdef.h"
 #include "thash.h"
 #include "tlist.h"

source/client/src/clientMain.c

@@ -20,13 +20,13 @@
 #include "functionMgt.h"
 #include "os.h"
 #include "query.h"
-#include "qworker.h"
 #include "scheduler.h"
 #include "tglobal.h"
 #include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
 #include "version.h"
+#include "tdatablock.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0

source/client/src/clientMsgHandler.c

@@ -20,6 +20,7 @@
 #include "query.h"
 #include "tdef.h"
 #include "tname.h"
+#include "tdatablock.h"
 #include "systable.h"
 
 static void setErrno(SRequestObj* pRequest, int32_t code) {

source/common/src/tdatablock.c

@@ -19,6 +19,8 @@
 #include "tlog.h"
 #include "tname.h"
 
+#define MALLOC_ALIGN_BYTES  32
+
 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
   ASSERT(pColumnInfoData != NULL);
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@@ -1174,6 +1176,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
     pColumn->varmeta.offset = (int32_t*)tmp;
     memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows));
   } else {
+    // prepare for the null bitmap
     char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -1184,11 +1187,19 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
     memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
 
     ASSERT(pColumn->info.bytes);
-    tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes);
+
+    // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned
+    tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes);
     if (tmp == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
+    // copy back the existed data
+    if (pColumn->pData != NULL) {
+      memcpy(tmp, pColumn->pData, existedRows * pColumn->info.bytes);
+      taosMemoryFreeClear(pColumn->pData);
+    }
+
     pColumn->pData = tmp;
     if (clearPayload) {
       memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));
@@ -1311,12 +1322,9 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
     SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
-    if (pSrc->pData == NULL) {
-      continue;
-    }
-
     colDataAssign(pDst, pSrc, src->info.rows, &src->info);
   }
+
   uint32_t cap = dst->info.capacity;
   dst->info = src->info;
   dst->info.capacity = cap;

source/common/src/tglobal.c

@@ -15,9 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "tglobal.h"
-#include "tcompare.h"
 #include "tconfig.h"
-#include "tdatablock.h"
+#include "tmisce.h"
 #include "tgrant.h"
 #include "tlog.h"
 
@@ -120,6 +119,9 @@ int32_t tsMinIntervalTime = 1;
 // maximum memory allowed to be allocated for a single csv load (in MB)
 int32_t tsMaxMemUsedByInsert = 1024;
 
+float   tsSelectivityRatio = 1.0;
+int32_t tsTagFilterResCacheSize = 1024*10;
+
 // the maximum allowed query buffer size during query processing for each data node.
 // -1 no limit (default)
 // 0  no query allowed, queries are disabled
@@ -318,7 +320,14 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1;
   if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1;
   if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1;
-  if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
+  if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1;
+
+  if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "SIMD-Supported", tsSIMDEnable, 0) != 0) return -1;
+
   if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;

source/common/src/tmisce.c

@@ -14,7 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "tdatablock.h"
+#include "tmisce.h"
 #include "tglobal.h"
 #include "tlog.h"
 #include "tname.h"

source/dnode/mgmt/node_util/src/dmEps.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "dmUtil.h"
+#include "tmisce.h"
 
 static void dmPrintEps(SDnodeData *pData);
 static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep);

source/dnode/mgmt/test/sut/src/client.cpp

@@ -15,6 +15,7 @@
 
 #include "sut.h"
 #include "tdatablock.h"
+#include "tmisce.h"
 
 static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
   TestClient* client = (TestClient*)parent;

source/dnode/mnode/impl/src/mndDnode.c

@@ -24,6 +24,7 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
+#include "tmisce.h"
 
 #define TSDB_DNODE_VER_NUMBER   1
 #define TSDB_DNODE_RESERVE_SIZE 64

source/dnode/mnode/impl/src/mndMnode.c

@@ -20,7 +20,7 @@
 #include "mndShow.h"
 #include "mndSync.h"
 #include "mndTrans.h"
-#include "mndUser.h"
+#include "tmisce.h"
 
 #define MNODE_VER_NUMBER   1
 #define MNODE_RESERVE_SIZE 64

source/dnode/mnode/impl/src/mndVgroup.c

@@ -22,6 +22,7 @@
 #include "mndShow.h"
 #include "mndTrans.h"
 #include "mndUser.h"
+#include "tmisce.h"
 
 #define VGROUP_VER_NUMBER   1
 #define VGROUP_RESERVE_SIZE 64

source/dnode/vnode/inc/vnode.h

@@ -99,15 +99,21 @@ void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
 void        metaReaderReleaseLock(SMetaReader *pReader);
 void        metaReaderClear(SMetaReader *pReader);
 int32_t     metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t     metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
 int         metaGetTableEntryByName(SMetaReader *pReader, const char *name);
 int32_t     metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags);
 int32_t     metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags);
 int32_t     metaReadNext(SMetaReader *pReader);
 const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
 int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
-int         metaGetTableUidByName(void *meta, char *tbName, int64_t *uid);
+int         metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
 int         metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
 bool        metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+int32_t     metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+                                      bool *acquired);
+int32_t     metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+                                  int32_t payloadLen, double selectivityRatio);
+int32_t     metaUidCacheClear(SMeta *pMeta, uint64_t suid);
 
 typedef struct SMetaFltParam {
   tb_uid_t suid;

source/dnode/vnode/src/inc/meta.h

@@ -70,7 +70,8 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid);
 int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo);
 int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid);
 int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
-void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta);
+void    metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta);
+int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle);
 
 struct SMeta {
   TdThreadRwlock lock;

source/dnode/vnode/src/inc/tsdb.h

@@ -185,6 +185,8 @@ int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItem
                        int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
 int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
 int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
+int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *),
+                        SArray **ppArray);
 // other
 int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
 void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -142,7 +142,7 @@ typedef struct SMetaInfo {
   int64_t version;
   int32_t skmVer;
 } SMetaInfo;
-int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo);
+int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
 
 typedef struct {
   int64_t uid;

source/dnode/vnode/src/meta/metaCache.c

@@ -31,6 +31,12 @@ typedef struct SMetaStbStatsEntry {
   SMetaStbStats              info;
 } SMetaStbStatsEntry;
 
+typedef struct STagFilterResEntry {
+  uint64_t suid;  // uid for super table
+  SList    list; // the linked list of md5 digest, extracted from the serialized tag query condition
+  uint32_t qTimes;// queried times for current super table
+} STagFilterResEntry;
+
 struct SMetaCache {
   // child, normal, super, table entry cache
   struct SEntryCache {
@@ -47,6 +53,11 @@ struct SMetaCache {
   } sStbStatsCache;
 
   // query cache
+  struct STagFilterResCache {
+    SHashObj*  pTableEntry;
+    SLRUCache* pUidResCache;
+    uint64_t   keyBuf[3];
+  } sTagFilterResCache;
 };
 
 static void entryCacheClose(SMeta* pMeta) {
@@ -79,6 +90,12 @@ static void statsCacheClose(SMeta* pMeta) {
   }
 }
 
+static void freeCacheEntryFp(void* param) {
+  STagFilterResEntry** p = param;
+  tdListEmpty(&(*p)->list);
+  taosMemoryFreeClear(*p);
+}
+
 int32_t metaCacheOpen(SMeta* pMeta) {
   int32_t     code = 0;
   SMetaCache* pCache = NULL;
@@ -109,9 +126,20 @@ int32_t metaCacheOpen(SMeta* pMeta) {
     goto _err2;
   }
 
-  pMeta->pCache = pCache;
+  pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5*1024*1024, -1, 0.5);
+  if (pCache->sTagFilterResCache.pUidResCache == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err2;
+  }
 
-_exit:
+  pCache->sTagFilterResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
+  if (pCache->sTagFilterResCache.pTableEntry == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err2;
+  }
+
+  taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
+  pMeta->pCache = pCache;
   return code;
 
 _err2:
@@ -119,7 +147,6 @@ _err2:
 
 _err:
   taosMemoryFree(pCache);
-
   metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   return code;
 }
@@ -128,6 +155,9 @@ void metaCacheClose(SMeta* pMeta) {
   if (pMeta->pCache) {
     entryCacheClose(pMeta);
     statsCacheClose(pMeta);
+
+    taosHashCleanup(pMeta->pCache->sTagFilterResCache.pTableEntry);
+    taosLRUCacheCleanup(pMeta->pCache->sTagFilterResCache.pUidResCache);
     taosMemoryFree(pMeta->pCache);
     pMeta->pCache = NULL;
   }
@@ -388,3 +418,142 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
 
   return code;
 }
+
+int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) {
+  uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf;
+
+  // generate the composed key for LRU cache
+  SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
+
+  pBuf[0] = suid;
+  memcpy(&pBuf[1], pKey, keyLen);
+
+  int32_t len = keyLen + sizeof(uint64_t);
+  LRUHandle *pHandle = taosLRUCacheLookup(pCache, pBuf, len);
+  if (pHandle == NULL) {
+    *acquireRes = 0;
+    return TSDB_CODE_SUCCESS;
+  } else {  // do some book mark work after acquiring the filter result from cache
+    STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
+    ASSERT(pEntry != NULL);
+    *acquireRes = 1;
+
+    const char* p = taosLRUCacheValue(pMeta->pCache->sTagFilterResCache.pUidResCache, pHandle);
+    int32_t size = *(int32_t*) p;
+    taosArrayAddBatch(pList1, p + sizeof(int32_t), size);
+
+    (*pEntry)->qTimes += 1;
+    taosLRUCacheRelease(pCache, pHandle, false);
+
+    // check if scanning all items are necessary or not
+    if ((*pEntry)->qTimes >= 5000 && TD_DLIST_NELES(&(*pEntry)->list) > 10) {
+      SArray* pList = taosArrayInit(64, POINTER_BYTES);
+
+      SListIter iter = {0};
+      tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
+
+      SListNode* pNode = NULL;
+      while ((pNode = tdListNext(&iter)) != NULL) {
+        memcpy(pBuf + sizeof(suid), pNode->data, keyLen);
+
+        // check whether it is existed in LRU cache, and remove it from linked list if not.
+        LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len);
+        if (pRes == NULL) {  // remove the item in the linked list
+          taosArrayPush(pList, &pNode);
+        } else {
+          taosLRUCacheRelease(pCache, pRes, false);
+        }
+      }
+
+      // remove the keys, of which query uid lists have been replaced already.
+      size_t s = taosArrayGetSize(pList);
+      for(int32_t i = 0; i < s; ++i) {
+        SListNode** p1 = taosArrayGet(pList, i);
+        tdListPopNode(&(*pEntry)->list, *p1);
+      }
+
+      (*pEntry)->qTimes = 0; // reset the query times
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void freePayload(const void* key, size_t keyLen, void* value) {
+  if (value == NULL) {
+    return;
+  }
+  taosMemoryFree(value);
+}
+
+// check both the payload size and selectivity ratio
+int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) {
+  if (selectivityRatio > tsSelectivityRatio) {
+    metaDebug("vgId:%d, suid:%" PRIu64
+              " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f",
+              TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio);
+    taosMemoryFree(pPayload);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (payloadLen > tsTagFilterResCacheSize) {
+    metaDebug("vgId:%d, suid:%" PRIu64
+              " failed to add to uid list cache, due to payload length %d greater than threshold %d",
+              TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize);
+    taosMemoryFree(pPayload);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache;
+  SHashObj*  pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry;
+
+  STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
+  if (pEntry == NULL) {
+    STagFilterResEntry* p = taosMemoryMalloc(sizeof(STagFilterResEntry));
+    p->qTimes = 0;
+    tdListInit(&p->list, keyLen);
+    taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES);
+    tdListAppend(&p->list, pKey);
+  } else {
+    tdListAppend(&(*pEntry)->list, pKey);
+  }
+
+  uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf;
+  pBuf[0] = suid;
+
+  memcpy(&pBuf[1], pKey, keyLen);
+  ASSERT(sizeof(uint64_t) + keyLen == 24);
+
+  // add to cache.
+  taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW);
+  metaDebug("vgId:%d, suid:%"PRIu64" list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode),
+            suid, (int32_t) taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+// remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables
+int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
+  STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t));
+  if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t keyLen = sizeof(uint64_t) * 3;
+  uint64_t p[3] = {0};
+  p[0] = suid;
+
+  SListIter iter = {0};
+  tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD);
+
+  SListNode* pNode = NULL;
+  while ((pNode = tdListNext(&iter)) != NULL) {
+    memcpy(&p[1], pNode->data, 16);
+    taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen);
+  }
+
+  (*pEntry)->qTimes = 0;
+  tdListEmpty(&(*pEntry)->list);
+
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/meta/metaQuery.c

@@ -152,7 +152,7 @@ bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
 }
 
 int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
-  SMeta  *pMeta = pReader->pMeta;
+  SMeta *pMeta = pReader->pMeta;
   int64_t version1;
 
   // query uid.idx
@@ -165,6 +165,18 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
   return metaGetTableEntryByVersion(pReader, version1, uid);
 }
 
+int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
+  SMeta *pMeta = pReader->pMeta;
+
+  SMetaInfo info;
+  if (metaGetInfo(pMeta, uid, &info, pReader) == TSDB_CODE_NOT_FOUND) {
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+    return -1;
+  }
+
+  return metaGetTableEntryByVersion(pReader, info.version, uid);
+}
+
 int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
   SMeta   *pMeta = pReader->pMeta;
   tb_uid_t uid;
@@ -211,7 +223,7 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
 
   return 0;
 }
-int metaGetTableUidByName(void *meta, char *tbName, int64_t *uid) {
+int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
   int         code = 0;
   SMetaReader mr = {0};
   metaReaderInit(&mr, (SMeta *)meta, 0);
@@ -614,7 +626,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
   SSkmDbKey skmDbKey;
   if (sver <= 0) {
     SMetaInfo info;
-    if (metaGetInfo(pMeta, suid ? suid : uid, &info) == 0) {
+    if (metaGetInfo(pMeta, suid ? suid : uid, &info, NULL) == 0) {
       sver = info.skmVer;
     } else {
       TBC *pSkmDbC = NULL;
@@ -1126,8 +1138,8 @@ int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
     valid = tdbTbcGet(pCursor->pCur, (const void **)pEntryKey, &nEntryKey, (const void **)&pEntryVal, &nEntryVal);
     if (valid < 0) break;
 
-    char   *pTableKey = (char *)pEntryKey;
-    int32_t cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
+    char *pTableKey = (char *)pEntryKey;
+    cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
     if (cmp == 0) {
       tb_uid_t tuid = *(tb_uid_t *)pEntryVal;
       taosArrayPush(pUids, &tuid);
@@ -1379,10 +1391,11 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj
 
 int32_t metaCacheGet(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo);
 
-int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) {
+int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo, SMetaReader *pReader) {
   int32_t code = 0;
   void   *pData = NULL;
   int     nData = 0;
+  int     lock = 0;
 
   metaRLock(pMeta);
 
@@ -1407,11 +1420,22 @@ int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) {
   pInfo->version = ((SUidIdxVal *)pData)->version;
   pInfo->skmVer = ((SUidIdxVal *)pData)->skmVer;
 
+  if (pReader != NULL) {
+    lock = !(pReader->flags & META_READER_NOLOCK);
+    if (lock) {
+      metaULock(pReader->pMeta);
+      // metaReaderReleaseLock(pReader);
+    }
+  }
   // upsert the cache
   metaWLock(pMeta);
   metaCacheUpsert(pMeta, pInfo);
   metaULock(pMeta);
 
+  if (lock) {
+    metaRLock(pReader->pMeta);
+  }
+
 _exit:
   tdbFree(pData);
   return code;

source/dnode/vnode/src/meta/metaSma.c

@@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   // validate req
   // save smaIndex
   metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
+  if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
 #if 1
     terrno = TSDB_CODE_TSMA_ALREADY_EXIST;
     metaReaderClear(&mr);

source/dnode/vnode/src/meta/metaTable.c

@@ -207,7 +207,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
     tb_uid_t uid = *(tb_uid_t *)pData;
     tdbFree(pData);
     SMetaInfo info;
-    metaGetInfo(pMeta, uid, &info);
+    metaGetInfo(pMeta, uid, &info, NULL);
     if (info.uid == info.suid) {
       return 0;
     } else {
@@ -458,6 +458,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
 
     metaWLock(pMeta);
     metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1);
+    metaUidCacheClear(pMeta, me.ctbEntry.suid);
     metaULock(pMeta);
   } else {
     me.ntbEntry.ctime = pReq->ctime;
@@ -681,6 +682,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     --pMeta->pVnode->config.vndStats.numOfCTables;
 
     metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1);
+    metaUidCacheClear(pMeta, e.ctbEntry.suid);
   } else if (e.type == TSDB_NORMAL_TABLE) {
     // drop schema.db (todo)
 
@@ -691,6 +693,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     // drop schema.db (todo)
 
     metaStatsCacheDrop(pMeta, uid);
+    metaUidCacheClear(pMeta, uid);
     --pMeta->pVnode->config.vndStats.numOfSTables;
   }
 
@@ -1074,6 +1077,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
               ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
 
+  metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid);
+
   metaULock(pMeta);
 
   tDecoderClear(&dc1);

source/dnode/vnode/src/sma/smaRollup.c

@@ -739,11 +739,13 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
     }
   }
 
-  tdBlockDataDestroy(pResList);
+  taosArrayDestroy(pResList);
+  qCleanExecTaskBlockBuf(taskInfo);
   return TSDB_CODE_SUCCESS;
 
 _err:
-  tdBlockDataDestroy(pResList);
+  taosArrayDestroy(pResList);
+  qCleanExecTaskBlockBuf(taskInfo);
   return TSDB_CODE_FAILED;
 }
 
@@ -921,7 +923,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, SMA_META(pSma), 0);
   smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid);
-  if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) {
+  if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
     smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid,
              terrstr());
     goto _err;
@@ -1125,7 +1127,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
   for (int64_t i = 0; i < arrSize; ++i) {
     tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
     smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
-    if (metaGetTableEntryByUid(&mr, suid) < 0) {
+    if (metaGetTableEntryByUidCache(&mr, suid) < 0) {
       smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
                terrstr());
       goto _err;

source/dnode/vnode/src/tq/tqExec.c

@@ -48,7 +48,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
   SMetaReader mr = {0};
   metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
   // TODO add reference to gurantee success
-  if (metaGetTableEntryByUid(&mr, uid) < 0) {
+  if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
     metaReaderClear(&mr);
     return -1;
   }

source/dnode/vnode/src/tq/tqRead.c

@@ -766,7 +766,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
         for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) {
           uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i);
 
-          int32_t code = metaGetTableEntryByUid(&mr, *id);
+          int32_t code = metaGetTableEntryByUidCache(&mr, *id);
           if (code != TSDB_CODE_SUCCESS) {
             qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
             continue;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -392,7 +392,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) {
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, uid) < 0) {
+  if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
     metaReaderClear(&mr);  // table not esist
     return 0;
   }

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -104,7 +104,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
   tb_uid_t   uid = pMsgIter->uid;
 
   SMetaInfo info;
-  code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info);
+  code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL);
   if (code) {
     code = TSDB_CODE_TDB_TABLE_NOT_EXIST;
     goto _err;
@@ -114,7 +114,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
     goto _err;
   }
   if (info.suid) {
-    metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info);
+    metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL);
   }
   if (pMsgIter->sversion != info.skmVer) {
     tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode),
@@ -153,7 +153,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
 
   // check if table exists
   SMetaInfo info;
-  code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info);
+  code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL);
   if (code) {
     code = TSDB_CODE_TDB_TABLE_NOT_EXIST;
     goto _err;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -38,7 +38,7 @@ typedef struct {
 typedef struct SBlockIndex {
   int32_t     ordinalIndex;
   int64_t     inFileOffset;
-  STimeWindow window;
+  STimeWindow window;  // todo replace it with overlap flag.
 } SBlockIndex;
 
 typedef struct STableBlockScanInfo {
@@ -551,7 +551,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
   }
 
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    SColumnInfoData colInfo = {0, {0}};
+    SColumnInfoData colInfo = {0};
     colInfo.info = pCond->colList[i];
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -709,11 +709,13 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
 
     tMapDataReset(&pScanInfo->mapData);
     tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData);
+    taosArrayEnsureCap(pScanInfo->pBlockList, pScanInfo->mapData.nItem);
 
     sizeInDisk += pScanInfo->mapData.nData;
+
+    SDataBlk block = {0};
     for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
-      SDataBlk block = {0};
-      tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk);
+      tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block);
 
       // 1. time range check
       if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) {
@@ -728,8 +730,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
       SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset};
       bIndex.window = (STimeWindow){.skey = block.minKey.ts, .ekey = block.maxKey.ts};
 
-      void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex);
-      if (p == NULL) {
+      void* p1 = taosArrayPush(pScanInfo->pBlockList, &bIndex);
+      if (p1 == NULL) {
         tMapDataClear(&pScanInfo->mapData);
         return TSDB_CODE_OUT_OF_MEMORY;
       }
@@ -752,6 +754,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
       numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
       pReader->idStr);
 
+
   pReader->cost.numOfBlocks += total;
   pReader->cost.headFileLoadTime += el;
 
@@ -1455,6 +1458,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+// todo: this attribute could be acquired during extractin the global ordered block list.
 static bool overlapWithNeighborBlock(SDataBlk* pBlock, SBlockIndex* pNeighborBlockIndex, int32_t order) {
   // it is the last block in current file, no chance to overlap with neighbor blocks.
   if (ASCENDING_TRAVERSE(order)) {
@@ -4346,7 +4350,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, pVnode->pMeta, 0);
-  int32_t code = metaGetTableEntryByUid(&mr, uid);
+  int32_t code = metaGetTableEntryByUidCache(&mr, uid);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     metaReaderClear(&mr);
@@ -4358,7 +4362,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
   if (mr.me.type == TSDB_CHILD_TABLE) {
     tDecoderClear(&mr.coder);
     *suid = mr.me.ctbEntry.suid;
-    code = metaGetTableEntryByUid(&mr, *suid);
+    code = metaGetTableEntryByUidCache(&mr, *suid);
     if (code != TSDB_CODE_SUCCESS) {
       terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
       metaReaderClear(&mr);

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -101,6 +101,30 @@ void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t
   tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem);
 }
 
+int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *),
+                        SArray **ppArray) {
+  int32_t code = 0;
+
+  SArray *pArray = taosArrayInit(pMapData->nItem, itemSize);
+  if (pArray == NULL) {
+    code = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  for (int32_t i = 0; i < pMapData->nItem; i++) {
+    tMapDataGetItemByIdx(pMapData, i, taosArrayReserve(pArray, 1), tGetItemFn);
+  }
+
+_exit:
+  if (code) {
+    *ppArray = NULL;
+    if (pArray) taosArrayDestroy(pArray);
+  } else {
+    *ppArray = pArray;
+  }
+  return code;
+}
+
 int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
   int32_t n = 0;
 

source/libs/executor/inc/executorimpl.h

@@ -153,12 +153,12 @@ typedef struct {
   SSchemaWrapper* qsw;
 } SSchemaInfo;
 
-typedef struct {
+typedef struct SExchangeOpStopInfo {
   int32_t operatorType;
   int64_t refId;
 } SExchangeOpStopInfo;
 
-typedef struct {
+typedef struct STaskStopInfo {
   SRWLatch lock;
   SArray*  pStopInfo;
 } STaskStopInfo;
@@ -181,6 +181,7 @@ struct SExecTaskInfo {
   SSubplan*             pSubplan;
   struct SOperatorInfo* pRoot;
   SLocalFetch           localFetch;
+  SArray*               pResultBlockList;// result block list
   STaskStopInfo         stopInfo;
 };
 
@@ -252,21 +253,22 @@ typedef struct SLimitInfo {
 } SLimitInfo;
 
 typedef struct SExchangeInfo {
-  SArray* pSources;
-  SArray* pSourceDataInfo;
-  tsem_t  ready;
-  void*   pTransporter;
+  SArray*             pSources;
+  SArray*             pSourceDataInfo;
+  tsem_t              ready;
+  void*               pTransporter;
+
   // SArray<SSDataBlock*>, result block list, used to keep the multi-block that
   // passed by downstream operator
   SArray*             pResultBlockList;
-  int32_t             rspBlockIndex;  // indicate the return block index in pResultBlockList
+  SArray*             pRecycledBlocks;// build a pool for small data block to avoid to repeatly create and then destroy.
   SSDataBlock*        pDummyBlock;    // dummy block, not keep data
   bool                seqLoadData;    // sequential load data or not, false by default
   int32_t             current;
   SLoadRemoteDataInfo loadInfo;
   uint64_t            self;
   SLimitInfo          limitInfo;
-  int64_t             openedTs;  // start exec time stamp
+  int64_t             openedTs;       // start exec time stamp, todo: move to SLoadRemoteDataInfo
 } SExchangeInfo;
 
 typedef struct SScanInfo {
@@ -301,9 +303,9 @@ typedef struct {
 } SAggOptrPushDownInfo;
 
 typedef struct STableMetaCacheInfo {
-  SLRUCache* pTableMetaEntryCache;  // 100 by default
-  uint64_t   metaFetch;
-  uint64_t   cacheHit;
+  SLRUCache*             pTableMetaEntryCache; // 100 by default
+  uint64_t               metaFetch;
+  uint64_t               cacheHit;
 } STableMetaCacheInfo;
 
 typedef struct STableScanBase {

source/libs/executor/src/cachescanoperator.c

@@ -44,6 +44,8 @@ static void         destroyCacheScanOperator(void* param);
 static int32_t      extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
 static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo);
 
+#define SCAN_ROW_TYPE(_t)  ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
+
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                            SExecTaskInfo* pTaskInfo) {
   int32_t           code = TSDB_CODE_SUCCESS;
@@ -75,32 +77,36 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
 
   STableListInfo* pTableList = pTaskInfo->pTableInfoList;
 
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
-  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+  int32_t totalTables = tableListGetSize(pTableList);
+  int32_t capacity = 0;
+
   pInfo->pUidList = taosArrayInit(4, sizeof(int64_t));
 
-  // partition by tbname, todo opt perf
-  if (oneTableForEachGroup(pTableList) || (tableListGetSize(pTableList) == 1)) {
-    pInfo->retrieveType =
-        CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
+  // partition by tbname
+  if (oneTableForEachGroup(pTableList) || (totalTables == 1)) {
+    pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull);
 
     STableKeyInfo* pList = tableListGetInfo(pTableList, 0);
 
-    size_t   num = tableListGetSize(pTableList);
     uint64_t suid = tableListGetSuid(pTableList);
-    code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
+    code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
                                    taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
 
+    capacity = TMIN(totalTables, 4096);
+
     pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
-    blockDataEnsureCapacity(pInfo->pBufferredRes, pOperator->resultInfo.capacity);
+    blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
   } else {  // by tags
-    pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE |
-                          (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW);
+    pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
+    capacity = 1;  // only one row output
   }
 
+  initResultSizeInfo(&pOperator->resultInfo, capacity);
+  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
   if (pScanNode->scan.pScanPseudoCols != NULL) {
     SExprSupp* p = &pInfo->pseudoExprSup;
     p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs);

source/libs/executor/src/exchangeoperator.c

@@ -181,10 +181,17 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-  size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList);
-  if (size == 0 || pExchangeInfo->rspBlockIndex >= size) {
-    pExchangeInfo->rspBlockIndex = 0;
-    taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock);
+  // we have buffered retrieved datablock, return it directly
+  SSDataBlock* p = NULL;
+  if (taosArrayGetSize(pExchangeInfo->pResultBlockList) > 0) {
+    p = taosArrayGetP(pExchangeInfo->pResultBlockList, 0);
+    taosArrayRemove(pExchangeInfo->pResultBlockList, 0);
+  }
+
+  if (p != NULL) {
+    taosArrayPush(pExchangeInfo->pRecycledBlocks, &p);
+    return p;
+  } else {
     if (pExchangeInfo->seqLoadData) {
       seqLoadRemoteData(pOperator);
     } else {
@@ -193,11 +200,13 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
 
     if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) {
       return NULL;
+    } else {
+      p = taosArrayGetP(pExchangeInfo->pResultBlockList, 0);
+      taosArrayRemove(pExchangeInfo->pResultBlockList, 0);
+      taosArrayPush(pExchangeInfo->pRecycledBlocks, &p);
+      return p;
     }
   }
-
-  // we have buffered retrieved datablock, return it directly
-  return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++);
 }
 
 static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
@@ -295,7 +304,8 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
 
   tsem_init(&pInfo->ready, 0, 0);
   pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
-  pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES);
+  pInfo->pResultBlockList = taosArrayInit(64, POINTER_BYTES);
+  pInfo->pRecycledBlocks = taosArrayInit(64, POINTER_BYTES);
 
   SExchangeOpStopInfo stopInfo = {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, pInfo->self};
   qAppendTaskStopInfo(pTaskInfo, &stopInfo);
@@ -342,10 +352,8 @@ void doDestroyExchangeOperatorInfo(void* param) {
   taosArrayDestroy(pExInfo->pSources);
   taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo);
 
-  if (pExInfo->pResultBlockList != NULL) {
-    taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock);
-    pExInfo->pResultBlockList = NULL;
-  }
+  taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock);
+  taosArrayDestroyEx(pExInfo->pRecycledBlocks, freeBlock);
 
   blockDataDestroy(pExInfo->pDummyBlock);
 
@@ -638,6 +646,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
 
     SRetrieveTableRsp*   pRsp = pDataInfo->pRsp;
     SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
+
     if (pRsp->numOfRows == 0) {
       qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
              ", totalRows:%" PRIu64 " try next",

source/libs/executor/src/executil.c

@@ -290,7 +290,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
   SMetaReader mr = {0};
 
   metaReaderInit(&mr, metaHandle, 0);
-  code = metaGetTableEntryByUid(&mr, info->uid);
+  code = metaGetTableEntryByUidCache(&mr, info->uid);
   if (TSDB_CODE_SUCCESS != code) {
     metaReaderClear(&mr);
     *pQualified = false;
@@ -405,7 +405,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
-  ctx.index = 0;
+
   ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
   if (ctx.cInfoList == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -421,7 +421,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
   }
 
   for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
-    SColumnInfoData colInfo = {0, {0}};
+    SColumnInfoData colInfo = {0};
     colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -582,7 +582,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
   }
 
   for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
-    SColumnInfoData colInfo = {0, {0}};
+    SColumnInfoData colInfo = {0};
     colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -964,67 +964,132 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
   return -1;
 }
 
+static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
+  if (pTagCond == NULL) {
+    return;
+  }
+
+  char*   payload = NULL;
+  int32_t len = 0;
+  nodesNodeToMsg(pTagCond, &payload, &len);
+
+  tMD5Init(pContext);
+  tMD5Update(pContext, (uint8_t*)payload, (uint32_t)len);
+  tMD5Final(pContext);
+
+  taosMemoryFree(payload);
+}
+
+static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle) {
+  if (pTagCond == NULL) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  terrno = TDB_CODE_SUCCESS;
+  SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
+  if (terrno != TDB_CODE_SUCCESS) {
+    colDataDestroy(pColInfoData);
+    taosMemoryFreeClear(pColInfoData);
+    taosArrayDestroy(res);
+    qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
+    return terrno;
+  }
+
+  int32_t i = 0;
+  int32_t len = taosArrayGetSize(res);
+
+  if (pColInfoData != NULL) {
+    bool*   pResult = (bool*)pColInfoData->pData;
+    SArray* p = taosArrayInit(taosArrayGetSize(res), sizeof(uint64_t));
+
+    while (i < len && pColInfoData) {
+      int64_t* uid = taosArrayGet(res, i);
+      qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, pResult[i]);
+
+      if (pResult[i]) {
+        taosArrayPush(p, uid);
+      }
+      i += 1;
+    }
+
+    taosArraySwap(res, p);
+    taosArrayDestroy(p);
+  }
+
+  colDataDestroy(pColInfoData);
+  taosMemoryFreeClear(pColInfoData);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
                      STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
+  size_t  numOfTables = 0;
 
   uint64_t tableUid = pScanNode->uid;
   pListInfo->suid = pScanNode->suid;
   SArray* res = taosArrayInit(8, sizeof(uint64_t));
 
-  if (pScanNode->tableType == TSDB_SUPER_TABLE) {
-    if (pTagIndexCond) {
-      SIndexMetaArg metaArg = {
-          .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
-
-      //      int64_t stt = taosGetTimestampUs();
-      SIdxFltStatus status = SFLT_NOT_INDEX;
-      code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
-      if (code != 0 || status == SFLT_NOT_INDEX) {
-        qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
-        code = TDB_CODE_SUCCESS;
-      }
-    } else if (!pTagCond) {
-      vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
-    }
-  } else {  // Create one table group.
+  if (pScanNode->tableType != TSDB_SUPER_TABLE) {
     if (metaIsTableExist(metaHandle, tableUid)) {
       taosArrayPush(res, &tableUid);
     }
-  }
 
-  if (pTagCond) {
-    terrno = TDB_CODE_SUCCESS;
-    SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
-    if (terrno != TDB_CODE_SUCCESS) {
-      colDataDestroy(pColInfoData);
-      taosMemoryFreeClear(pColInfoData);
-      taosArrayDestroy(res);
-      qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
-      return terrno;
+    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+  } else {
+    // try to retrieve the result from meta cache
+    T_MD5_CTX context = {0};
+    genTagFilterDigest(pTagCond, &context);
+
+    bool acquired = false;
+    metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), res, &acquired);
+    if (acquired) {
+      qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(res));
+      goto _end;
     }
 
-    int32_t i = 0;
-    int32_t j = 0;
-    int32_t len = taosArrayGetSize(res);
-    while (i < taosArrayGetSize(res) && j < len && pColInfoData) {
-      void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-
-      int64_t* uid = taosArrayGet(res, i);
-      qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, *(bool*)var);
-      if (*(bool*)var == false) {
-        taosArrayRemove(res, i);
-        j++;
-        continue;
+    if (!pTagCond) {  // no tag condition exists, let's fetch all tables of this super table
+      ASSERT(pTagIndexCond == NULL);
+      vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
+    } else {
+      // failed to find the result in the cache, let try to calculate the results
+      if (pTagIndexCond) {
+        SIndexMetaArg metaArg = {
+            .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
+
+        SIdxFltStatus status = SFLT_NOT_INDEX;
+        code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
+        if (code != 0 || status == SFLT_NOT_INDEX) {
+          qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
+          code = TDB_CODE_SUCCESS;
+        }
       }
-      i++;
-      j++;
     }
-    colDataDestroy(pColInfoData);
-    taosMemoryFreeClear(pColInfoData);
+
+    code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    // let's add the filter results into meta-cache
+    numOfTables = taosArrayGetSize(res);
+    size_t size = numOfTables * sizeof(uint64_t) + sizeof(int32_t);
+    char*  pPayload = taosMemoryMalloc(size);
+    *(int32_t*)pPayload = numOfTables;
+
+    if (numOfTables > 0) {
+      memcpy(pPayload + sizeof(int32_t), taosArrayGet(res, 0), numOfTables * sizeof(uint64_t));
+    }
+
+    metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
   }
 
-  size_t numOfTables = taosArrayGetSize(res);
+_end:
+  numOfTables = taosArrayGetSize(res);
   for (int i = 0; i < numOfTables; i++) {
     STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
 
@@ -1034,7 +1099,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
-    qDebug("tagfilter get uid:%" PRIu64 "", info.uid);
+    qTrace("tagfilter get uid:%" PRIu64 "", info.uid);
   }
 
   taosArrayDestroy(res);
@@ -1057,7 +1122,7 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
 int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, uid) != 0) {  // table not exist
+  if (metaGetTableEntryByUidCache(&mr, uid) != 0) {  // table not exist
     metaReaderClear(&mr);
     return TSDB_CODE_PAR_TABLE_NOT_EXIST;
   }
@@ -1770,7 +1835,7 @@ STableListInfo* tableListCreate() {
     goto _error;
   }
 
-  pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
+  pListInfo->map = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
   if (pListInfo->map == NULL) {
     goto _error;
   }
@@ -1914,13 +1979,15 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return code;
   }
 
+  int32_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
   ASSERT(pTableListInfo->numOfOuputGroups == 1);
 
   int64_t st1 = taosGetTimestampUs();
   pTaskInfo->cost.extractListTime = (st1 - st) / 1000.0;
-  qDebug("extract queried table list completed, elapsed time:%.2f ms %s", pTaskInfo->cost.extractListTime, idStr);
+  qDebug("extract queried table list completed, %d tables, elapsed time:%.2f ms %s", numOfTables,
+         pTaskInfo->cost.extractListTime, idStr);
 
-  if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
+  if (numOfTables == 0) {
     qDebug("no table qualified for query, %s" PRIx64, idStr);
     return TSDB_CODE_SUCCESS;
   }

source/libs/executor/src/executor.c

@@ -493,7 +493,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
     memcpy(&pTaskInfo->localFetch, pLocal, sizeof(*pLocal));
   }
 
-  taosArrayClearEx(pResList, freeBlock);
+  taosArrayClear(pResList);
 
   int64_t curOwner = 0;
   if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
@@ -531,8 +531,20 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
 
   int64_t st = taosGetTimestampUs();
 
+  int32_t blockIndex = 0;
   while ((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) {
-    SSDataBlock* p = createOneDataBlock(pRes, true);
+    SSDataBlock* p = NULL;
+    if (blockIndex >= taosArrayGetSize(pTaskInfo->pResultBlockList)) {
+      SSDataBlock* p1 = createOneDataBlock(pRes, true);
+      taosArrayPush(pTaskInfo->pResultBlockList, &p1);
+      p = p1;
+    } else {
+      p = *(SSDataBlock**) taosArrayGet(pTaskInfo->pResultBlockList, blockIndex);
+      copyDataBlock(p, pRes);
+    }
+
+    blockIndex += 1;
+
     current += p->info.rows;
     ASSERT(p->info.rows > 0);
     taosArrayPush(pResList, &p);
@@ -560,6 +572,18 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
   return pTaskInfo->code;
 }
 
+void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  SArray* pList = pTaskInfo->pResultBlockList;
+  size_t num = taosArrayGetSize(pList);
+  for(int32_t i = 0; i < num; ++i) {
+    SSDataBlock** p = taosArrayGet(pTaskInfo->pResultBlockList, i);
+    blockDataDestroy(*p);
+  }
+
+  taosArrayClear(pTaskInfo->pResultBlockList);
+}
+
 int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   int64_t        threadId = taosGetSelfPthreadId();

source/libs/executor/src/executorimpl.c

@@ -246,7 +246,7 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes
 
   // in the first scan, new space needed for results
   int32_t pageId = -1;
-  SIDList list = getDataBufPagesIdList(pResultBuf);
+  SArray* list = getDataBufPagesIdList(pResultBuf);
 
   if (taosArrayGetSize(list) == 0) {
     pData = getNewBufPage(pResultBuf, &pageId);
@@ -305,13 +305,13 @@ typedef struct {
 } SFunctionCtxStatus;
 
 static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
-  pStatus->hasAgg = pCtx->input.colDataAggIsSet;
+  pStatus->hasAgg = pCtx->input.colDataSMAIsSet;
   pStatus->numOfRows = pCtx->input.numOfRows;
   pStatus->startOffset = pCtx->input.startRowIndex;
 }
 
 static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) {
-  pCtx->input.colDataAggIsSet = pStatus->hasAgg;
+  pCtx->input.colDataSMAIsSet = pStatus->hasAgg;
   pCtx->input.numOfRows = pStatus->numOfRows;
   pCtx->input.startRowIndex = pStatus->startOffset;
 }
@@ -328,8 +328,8 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfo
 
     // not a whole block involved in query processing, statistics data can not be used
     // NOTE: the original value of isSet have been changed here
-    if (pCtx[k].input.colDataAggIsSet && forwardStep < numOfTotal) {
-      pCtx[k].input.colDataAggIsSet = false;
+    if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) {
+      pCtx[k].input.colDataSMAIsSet = false;
     }
 
     if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) {
@@ -439,7 +439,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
 
     SInputColumnInfoData* pInput = &pCtx[i].input;
     pInput->uid = pBlock->info.uid;
-    pInput->colDataAggIsSet = false;
+    pInput->colDataSMAIsSet = false;
 
     SExprInfo* pOneExpr = &pExprSup->pExprInfo[i];
     for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
@@ -755,7 +755,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
   pInput->totalRows = numOfRows;
 
   if (pBlock->pBlockAgg != NULL) {
-    pInput->colDataAggIsSet = true;
+    pInput->colDataSMAIsSet = true;
 
     for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) {
       SFunctParam* pFuncParam = &pExprInfo->base.pParam[j];
@@ -764,7 +764,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
         int32_t slotId = pFuncParam->pCol->slotId;
         pInput->pColumnDataAgg[j] = pBlock->pBlockAgg[slotId];
         if (pInput->pColumnDataAgg[j] == NULL) {
-          pInput->colDataAggIsSet = false;
+          pInput->colDataSMAIsSet = false;
         }
 
         // Here we set the column info data since the data type for each column data is required, but
@@ -775,7 +775,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
       }
     }
   } else {
-    pInput->colDataAggIsSet = false;
+    pInput->colDataSMAIsSet = false;
   }
 }
 
@@ -2300,6 +2300,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
   pTaskInfo->execModel = model;
   pTaskInfo->pTableInfoList = tableListCreate();
   pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
+  pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
 
   char* p = taosMemoryCalloc(1, 128);
   snprintf(p, 128, "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, queryId);
@@ -2313,7 +2314,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
 int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, pHandle->meta, 0);
-  int32_t code = metaGetTableEntryByUid(&mr, pScanNode->uid);
+  int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid,
            GET_TASKID(pTaskInfo));
@@ -2332,7 +2333,7 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode,
     tDecoderClear(&mr.coder);
 
     tb_uid_t suid = mr.me.ctbEntry.suid;
-    metaGetTableEntryByUid(&mr, suid);
+    metaGetTableEntryByUidCache(&mr, suid);
     pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow);
     pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version;
   } else {
@@ -2901,6 +2902,11 @@ _complete:
   return terrno;
 }
 
+static void freeBlock(void* pParam) {
+  SSDataBlock* pBlock = *(SSDataBlock**)pParam;
+  blockDataDestroy(pBlock);
+}
+
 void doDestroyTask(SExecTaskInfo* pTaskInfo) {
   qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
 
@@ -2913,6 +2919,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) {
     nodesDestroyNode((SNode*)pTaskInfo->pSubplan);
   }
 
+  taosArrayDestroyEx(pTaskInfo->pResultBlockList, freeBlock);
   taosArrayDestroy(pTaskInfo->stopInfo.pStopInfo);
   taosMemoryFreeClear(pTaskInfo->sql);
   taosMemoryFreeClear(pTaskInfo->id.str);

source/libs/executor/src/groupoperator.c

@@ -274,10 +274,9 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
   //    return;
   //  }
 
-  int32_t     len = 0;
-  STimeWindow w = TSWINDOW_INITIALIZER;
-
+  int32_t len = 0;
   terrno = TSDB_CODE_SUCCESS;
+
   int32_t num = 0;
   for (int32_t j = 0; j < pBlock->info.rows; ++j) {
     // Compare with the previous row of this column, and do not set the output buffer again if they are identical.

source/libs/executor/src/scanoperator.c

@@ -484,7 +484,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
   // 1. check if it is existed in meta cache
   if (pCache == NULL) {
     metaReaderInit(&mr, pHandle->meta, 0);
-    code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
+    code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
     if (code != TSDB_CODE_SUCCESS) {
       if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
         qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid,
@@ -508,7 +508,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
     h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid));
     if (h == NULL) {
       metaReaderInit(&mr, pHandle->meta, 0);
-      code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
+      code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
       if (code != TSDB_CODE_SUCCESS) {
         if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
           qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",

source/libs/executor/src/sortoperator.c

@@ -551,6 +551,7 @@ typedef struct SMultiwayMergeOperatorInfo {
   SSortHandle*   pSortHandle;
   SColMatchInfo  matchInfo;
   SSDataBlock*   pInputBlock;
+  SSDataBlock*   pIntermediateBlock;   // to hold the intermediate result
   int64_t        startTs;  // sort start time
   bool           groupSort;
   bool           hasGroupId;
@@ -651,12 +652,19 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   blockDataCleanup(pDataBlock);
 
-  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
-  if (p == NULL) {
-    return NULL;
+  if (pInfo->pIntermediateBlock == NULL) {
+    pInfo->pIntermediateBlock = tsortGetSortedDataBlock(pHandle);
+    if (pInfo->pIntermediateBlock == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+    blockDataEnsureCapacity(pInfo->pIntermediateBlock, capacity);
+  } else {
+    blockDataCleanup(pInfo->pIntermediateBlock);
   }
 
-  blockDataEnsureCapacity(p, capacity);
+  SSDataBlock* p = pInfo->pIntermediateBlock;
+
   while (1) {
     doGetSortedBlockData(pInfo, pHandle, capacity, p);
     if (p->info.rows == 0) {
@@ -686,7 +694,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
     pDataBlock->info.groupId = pInfo->groupId;
   }
 
-  blockDataDestroy(p);
   qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
          pDataBlock->info.rows);
 
@@ -722,6 +729,7 @@ void destroyMultiwayMergeOperatorInfo(void* param) {
   SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
   pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
+  pInfo->pIntermediateBlock = blockDataDestroy(pInfo->pIntermediateBlock);
 
   tsortDestroySortHandle(pInfo->pSortHandle);
   taosArrayDestroy(pInfo->pSortInfo);

source/libs/executor/src/sysscanoperator.c

@@ -441,6 +441,9 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
     if (code != TSDB_CODE_SUCCESS) {
       // terrno has been set by metaGetTableEntryByName, therefore, return directly
+      metaReaderClear(&smrChildTable);
+      blockDataDestroy(dataBlock);
+      pInfo->loadInfo.totalRows = 0;
       return NULL;
     }
 
@@ -456,12 +459,16 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
     if (code != TSDB_CODE_SUCCESS) {
       // terrno has been set by metaGetTableEntryByUid
+      metaReaderClear(&smrSuperTable);
+      metaReaderClear(&smrChildTable);
+      blockDataDestroy(dataBlock);
       return NULL;
     }
 
     sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
     metaReaderClear(&smrSuperTable);
     metaReaderClear(&smrChildTable);
+
     if (numOfRows > 0) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
       numOfRows = 0;

source/libs/executor/src/tsort.c

@@ -801,7 +801,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) {
   index = tMergeTreeGetChosenIndex(pHandle->pMergeTree);
   pSource = pHandle->cmpParam.pSources[index];
 
-  assert(pSource->src.pBlock != NULL);
+  ASSERT(pSource->src.pBlock != NULL);
 
   pHandle->tupleHandle.rowIndex = pSource->src.rowIndex;
   pHandle->tupleHandle.pBlock = pSource->src.pBlock;

source/libs/function/CMakeLists.txt

 aux_source_directory(src FUNCTION_SRC)
+aux_source_directory(src/detail FUNCTION_SRC_DETAIL)
 list(REMOVE_ITEM FUNCTION_SRC src/udfd.c)
-add_library(function STATIC ${FUNCTION_SRC})
+add_library(function STATIC ${FUNCTION_SRC} ${FUNCTION_SRC_DETAIL})
 target_include_directories(
     function
     PUBLIC

source/libs/function/inc/builtinsimpl.h

@@ -23,6 +23,32 @@ extern "C" {
 #include "function.h"
 #include "functionMgt.h"
 
+typedef struct SSumRes {
+  union {
+    int64_t  isum;
+    uint64_t usum;
+    double   dsum;
+  };
+  int16_t type;
+  int64_t prevTs;
+  bool    isPrevTsSet;
+} SSumRes;
+
+typedef struct SMinmaxResInfo {
+  bool      assign;  // assign the first value or not
+  int64_t   v;
+  STuplePos tuplePos;
+
+  STuplePos nullTuplePos;
+  bool      nullTupleSaved;
+  int16_t   type;
+} SMinmaxResInfo;
+int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc);
+
+STuplePos   saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey);
+int32_t     updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
+const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);
+
 bool    functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
@@ -119,15 +145,10 @@ EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow);
 int32_t lastRowFunction(SqlFunctionCtx* pCtx);
 
 bool    getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
-bool    getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 bool    topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t topFunction(SqlFunctionCtx* pCtx);
-int32_t topFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t bottomFunction(SqlFunctionCtx* pCtx);
-int32_t bottomFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
-int32_t topBotPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
-int32_t topBotMergeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 int32_t topCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 int32_t getTopBotInfoSize(int64_t numOfItems);

source/libs/function/src/tpercentile.c

@@ -494,7 +494,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
         resetSlotInfo(pMemBucket);
 
         int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
-        SIDList list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId));
+        SArray* list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId));
         ASSERT(list != NULL && list->size > 0);
 
         for (int32_t f = 0; f < list->size; ++f) {

source/libs/function/src/udfd.c

@@ -27,6 +27,7 @@
 #include "tglobal.h"
 #include "tmsg.h"
 #include "trpc.h"
+#include "tmisce.h"
 // clang-foramt on
 
 typedef struct SUdfdContext {

source/libs/parser/test/mockCatalogService.cpp

@@ -20,9 +20,9 @@
 #include <map>
 #include <set>
 
-#include "tdatablock.h"
 #include "tname.h"
 #include "ttypes.h"
+#include "tmisce.h"
 
 std::unique_ptr<MockCatalogService> g_mockCatalogService;
 

source/libs/qworker/src/qwUtil.c

@@ -275,7 +275,7 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) {
   qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
   if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
     qDestroyTask(otaskHandle);
-    qDebug("task handle destryed");
+    qDebug("task handle destroyed");
   }
 }
 
@@ -308,7 +308,7 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) {
   if (ctx->sinkHandle) {
     dsDestroyDataSinker(ctx->sinkHandle);
     ctx->sinkHandle = NULL;
-    qDebug("sink handle destryed");
+    qDebug("sink handle destroyed");
   }
 }
 

source/libs/qworker/src/qworker.c

@@ -18,11 +18,6 @@ SQWorkerMgmt gQwMgmt = {
     .qwNum = 0,
 };
 
-static void freeBlock(void *param) {
-  SSDataBlock *pBlock = *(SSDataBlock **)param;
-  blockDataDestroy(pBlock);
-}
-
 int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
   int32_t         code = 0;
   SSchedulerHbRsp rsp = {0};
@@ -203,7 +198,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
   }
 
 _return:
-  taosArrayDestroyEx(pResList, freeBlock);
+  taosArrayDestroy(pResList);
   QW_RET(code);
 }
 

source/libs/scheduler/test/schedulerTests.cpp

@@ -38,6 +38,7 @@
 #include "tglobal.h"
 #include "trpc.h"
 #include "tvariant.h"
+#include "tmisce.h"
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wwrite-strings"

source/libs/transport/test/transUT.cpp

@@ -20,6 +20,8 @@
 #include "tlog.h"
 #include "transLog.h"
 #include "trpc.h"
+#include "tmisce.h"
+
 using namespace std;
 
 const char *label = "APP";

source/os/src/osEnv.c

@@ -37,6 +37,12 @@ float           tsNumOfCores = 0;
 int64_t         tsTotalMemoryKB = 0;
 char           *tsProcPath = NULL;
 
+char            tsSIMDEnable = 0;
+char            tsSSE42Enable = 0;
+char            tsAVXEnable = 0;
+char            tsAVX2Enable = 0;
+char            tsFMAEnable = 0;
+
 void osDefaultInit() {
   taosSeedRand(taosSafeRand());
   taosGetSystemLocale(tsLocale, tsCharset);
@@ -99,7 +105,7 @@ bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reser
 
 bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; }
 
-void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); }
+void osSetTimezone(const char *tz) { taosSetSystemTimezone(tz, tsTimezoneStr, &tsDaylight, &tsTimezone); }
 
 void osSetSystemLocale(const char *inLocale, const char *inCharSet) {
   memcpy(tsLocale, inLocale, strlen(inLocale) + 1);

source/os/src/osFile.c

@@ -774,6 +774,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) {
   return getline(ptrBuf, &len, pFile->fp);
 #endif
 }
+
 int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) {
   if (pFile == NULL || buf == NULL) {
     return -1;
@@ -784,6 +785,7 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) {
   }
   return strlen(buf);
 }
+
 int32_t taosEOFFile(TdFilePtr pFile) {
   if (pFile == NULL) {
     return 0;

source/os/src/osLocale.c

@@ -67,6 +67,9 @@ char *taosCharsetReplace(char *charsetstr) {
 }
 
 /**
+ * TODO: here we may employ the systemctl API to set/get the correct locale on the Linux. In some cases, the setlocale
+ *  seems does not response as expected.
+ *
  * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
  * both the TDengine Server and the Client may be interrupted.
  *
@@ -148,7 +151,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
    *
    * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
    *
-   * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
+   * If user does not specify the locale in taos.cfg, the program then uses default LC_CTYPE as system locale.
    *
    * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
    * for libiconv that is employed to convert string in this system. This program will automatically use

source/os/src/osMemory.c

@@ -345,3 +345,15 @@ void taosMemoryTrim(int32_t size) {
   malloc_trim(size);
 #endif
 }
+
+void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
+#ifdef USE_TD_MEMORY
+  ASSERT(0);
+#else
+#if defined(LINUX)
+  return memalign(alignment, size);
+#else
+  return taosMemoryMalloc(size);
+#endif
+#endif
+}

source/os/src/osSysinfo.c

@@ -155,8 +155,8 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
   }
 
   char    line[1024];
-  ssize_t _bytes = taosGetsFile(pFile, sizeof(line), line);
-  if ((_bytes < 0) || (line == NULL)) {
+  ssize_t bytes = taosGetsFile(pFile, sizeof(line), line);
+  if (bytes < 0) {
     taosCloseFile(&pFile);
     return -1;
   }
@@ -193,9 +193,9 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
     return -1;
   }
 
-  char    line[1024];
-  ssize_t _bytes = taosGetsFile(pFile, sizeof(line), line);
-  if ((_bytes < 0) || (line == NULL)) {
+  char    line[1024] = {0};
+  ssize_t bytes = taosGetsFile(pFile, sizeof(line), line);
+  if (bytes < 0) {
     taosCloseFile(&pFile);
     return -1;
   }
@@ -239,6 +239,7 @@ void taosGetSystemInfo() {
   taosGetCpuCores(&tsNumOfCores);
   taosGetTotalMemory(&tsTotalMemoryKB);
   taosGetCpuUsage(NULL, NULL);
+  taosGetCpuInstructions(&tsSSE42Enable, &tsAVXEnable, &tsAVX2Enable, &tsFMAEnable);
 #endif
 }
 
@@ -366,7 +367,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
 
   return code;
 #else
-  char    line[1024];
+  char    line[1024] = {0};
   size_t  size = 0;
   int32_t done = 0;
   int32_t code = -1;
@@ -468,6 +469,46 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) {
   }
 }
 
+#define __cpuid_fix(level, a, b, c, d) \
+              __asm__("xor %%ecx, %%ecx\n" \
+                      "cpuid\n" \
+                      : "=a"(a), "=b"(b), "=c"(c), "=d"(d) \
+                      : "0"(level))
+
+// todo add for windows and mac
+int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) {
+#ifdef WINDOWS
+#elif defined(_TD_DARWIN_64)
+#else
+
+  // Since the compiler is not support avx/avx2 instructions, the global variables always need to be
+  // set to be false
+#if __AVX__ || __AVX2__
+  tsSIMDEnable = true;
+#else
+  tsSIMDEnable = false;
+#endif
+
+  uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
+
+  int32_t ret = __get_cpuid(1, &eax, &ebx, &ecx, &edx);
+  if (ret == 0) {
+    return -1;  // failed to get the cpuid info
+  }
+
+  *sse42 = (char) ((ecx & bit_SSE4_2) == bit_SSE4_2);
+  *avx   = (char) ((ecx & bit_AVX) == bit_AVX);
+  *fma   = (char) ((ecx & bit_FMA) == bit_FMA);
+
+  // work around a bug in GCC.
+  // Ref to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756
+  __cpuid_fix(7u, eax, ebx, ecx, edx);
+  *avx2 = (char) ((ebx & bit_AVX2) == bit_AVX2);
+#endif
+
+  return 0;
+}
+
 int32_t taosGetTotalMemory(int64_t *totalKB) {
 #ifdef WINDOWS
   MEMORYSTATUSEX memsStat;
@@ -511,11 +552,11 @@ int32_t taosGetProcMemory(int64_t *usedKB) {
     return -1;
   }
 
-  ssize_t _bytes = 0;
-  char    line[1024];
+  ssize_t bytes = 0;
+  char    line[1024] = {0};
   while (!taosEOFFile(pFile)) {
-    _bytes = taosGetsFile(pFile, sizeof(line), line);
-    if ((_bytes < 0) || (line == NULL)) {
+    bytes = taosGetsFile(pFile, sizeof(line), line);
+    if (bytes < 0) {
       break;
     }
     if (strstr(line, "VmRSS:") != NULL) {
@@ -523,7 +564,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) {
     }
   }
 
-  if (line == NULL) {
+  if (strlen(line) < 0) {
     // printf("read file:%s failed", tsProcMemFile);
     taosCloseFile(&pFile);
     return -1;
@@ -624,14 +665,14 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int
   TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM);
   if (pFile == NULL) return -1;
 
-  ssize_t _bytes = 0;
-  char    line[1024];
+  ssize_t bytes = 0;
+  char    line[1024] = {0};
   char    tmp[24];
   int     readIndex = 0;
 
   while (!taosEOFFile(pFile)) {
-    _bytes = taosGetsFile(pFile, sizeof(line), line);
-    if (_bytes < 10 || line == NULL) {
+    bytes = taosGetsFile(pFile, sizeof(line), line);
+    if (bytes < 10) {
       break;
     }
     if (strstr(line, "rchar:") != NULL) {

source/os/src/osTime.c

@@ -339,7 +339,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) {
 #endif
 }
 
-FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) {
+int32_t taosGetTimeOfDay(struct timeval *tv) {
 #ifdef WINDOWS
   time_t t;
   t = taosGetTimestampSec();
@@ -455,6 +455,7 @@ static int isLeapYear(time_t year) {
   else
     return 1;
 }
+
 struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) {
   if (result == NULL) {
     return localtime(timep);
@@ -542,7 +543,9 @@ struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst)
 #endif
   return result;
 }
+
 int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
+
 int32_t taosClockGetTime(int clock_id, struct timespec *pTS) {
 #ifdef WINDOWS
   LARGE_INTEGER        t;

source/util/src/tarray.c

@@ -17,6 +17,8 @@
 #include "tarray.h"
 #include "tcoding.h"
 
+// todo refactor API
+
 SArray* taosArrayInit(size_t size, size_t elemSize) {
   assert(elemSize > 0);
 
@@ -181,6 +183,17 @@ void* taosArrayAddAll(SArray* pArray, const SArray* pInput) {
   }
 }
 
+void* taosArrayReserve(SArray* pArray, int32_t num) {
+  if (taosArrayEnsureCap(pArray, pArray->size + num) != 0) {
+    return NULL;
+  }
+
+  void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
+  pArray->size += num;
+
+  return dst;
+}
+
 void* taosArrayPop(SArray* pArray) {
   assert(pArray != NULL);
 

source/util/src/tconfig.c

@@ -564,13 +564,13 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
     if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
     if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue;
     tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
-    for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) {
-      if (src[i] == 0) src[i] = ' ';
+    for (int32_t j = 0; j < CFG_SRC_PRINT_LEN; ++j) {
+      if (src[j] == 0) src[j] = ' ';
     }
 
     tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN);
-    for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) {
-      if (name[i] == 0) name[i] = ' ';
+    for (int32_t j = 0; j < CFG_NAME_PRINT_LEN; ++j) {
+      if (name[j] == 0) name[j] = ' ';
     }
 
     switch (pItem->dtype) {

source/util/src/tcrc32c.c

@@ -24,7 +24,6 @@
 #endif
 
 #include "tcrc32c.h"
-#include "tdef.h"
 
 #define POLY        0x82f63b78
 #define LONG_SHIFT  8192

source/util/src/thash.c

@@ -67,7 +67,7 @@ struct SHashObj {
   bool              enableUpdate;  // enable update
   SArray           *pMemBlock;     // memory block allocated for SHashEntry
   _hash_before_fn_t callbackFp;    // function invoked before return the value to caller
-  int64_t           compTimes;
+//  int64_t           compTimes;
 };
 
 /*
@@ -889,4 +889,4 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
 
 void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); }
 
-int64_t taosHashGetCompTimes(SHashObj *pHashObj) { return atomic_load_64(&pHashObj->compTimes); }
+int64_t taosHashGetCompTimes(SHashObj *pHashObj) { return 0 /*atomic_load_64(&pHashObj->compTimes)*/; }