Merge remote-tracking branch 'origin/3.0' into feature/config

contrib/test/craft/CMakeLists.txt

 add_executable(simulate_vnode "simulate_vnode.c")
-target_link_libraries(simulate_vnode craft lz4 uv_a)
\ No newline at end of file
+target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
\ No newline at end of file

contrib/test/traft/single_node/CMakeLists.txt

@@ -3,4 +3,4 @@ target_sources(singleNode
   PRIVATE
   "singleNode.c" 
 )
-target_link_libraries(singleNode traft lz4 uv_a)
+target_link_libraries(singleNode PUBLIC traft lz4 uv_a)

include/os/os.h

@@ -44,6 +44,7 @@ extern "C" {
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/utsname.h>
+#include <sys/param.h>
 #include <unistd.h>
 #include <wchar.h>
 #include <wctype.h>

include/util/tcache.h

@@ -40,55 +40,9 @@ typedef struct SCacheStatis {
   int64_t refreshCount;
 } SCacheStatis;
 
-struct STrashElem;
-
-typedef struct SCacheDataNode {
-  uint64_t           addedTime;   // the added time when this element is added or updated into cache
-  uint64_t           lifespan;    // life duration when this element should be remove from cache
-  uint64_t           expireTime;  // expire time
-  uint64_t           signature;
-  struct STrashElem *pTNodeHeader;    // point to trash node head
-  uint16_t           keySize : 15;    // max key size: 32kb
-  bool               inTrashcan : 1;  // denote if it is in trash or not
-  uint32_t           size;            // allocated size for current SCacheDataNode
-  T_REF_DECLARE()
-  char *key;
-  char  data[];
-} SCacheDataNode;
-
-typedef struct STrashElem {
-  struct STrashElem *prev;
-  struct STrashElem *next;
-  SCacheDataNode    *pData;
-} STrashElem;
-
-/*
- * to accommodate the old data which has the same key value of new one in hashList
- * when an new node is put into cache, if an existed one with the same key:
- * 1. if the old one does not be referenced, update it.
- * 2. otherwise, move the old one to pTrash, addedTime the new one.
- *
- * when the node in pTrash does not be referenced, it will be release at the expired expiredTime
- */
-typedef struct {
-  int64_t           totalSize;  // total allocated buffer in this hash table, SCacheObj is not included.
-  int64_t           refreshTime;
-  STrashElem       *pTrash;
-  char             *name;
-  SCacheStatis      statistics;
-  SHashObj         *pHashTable;
-  __cache_free_fn_t freeFp;
-  uint32_t          numOfElemsInTrash;  // number of element in trash
-  uint8_t           deleting;           // set the deleting flag to stop refreshing ASAP.
-  pthread_t         refreshWorker;
-  bool              extendLifespan;  // auto extend life span when one item is accessed.
-  int64_t           checkTick;       // tick used to record the check times of the refresh threads
-#if defined(LINUX)
-  pthread_rwlock_t lock;
-#else
-  pthread_mutex_t lock;
-#endif
-} SCacheObj;
+typedef struct SCacheObj  SCacheObj;
+typedef struct SCacheIter SCacheIter;
+typedef struct STrashElem STrashElem;
 
 /**
  * initialize the cache object
@@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data);
  * @param data
  * @return
  */
-void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
+void *taosCacheTransferData(SCacheObj *pCacheObj, void **data);
 
 /**
  * remove data in cache, the data will not be removed immediately.
@@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
  */
 void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
 
+/**
+ *
+ * @param pCacheObj
+ * @return
+ */
+size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
+
 /**
  *  move all data node into trash, clear node in trash can if it is not referenced by any clients
  * @param handle
@@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
  */
 void taosStopCacheRefreshWorker();
 
+SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
+bool taosCacheIterNext(SCacheIter* pIter);
+void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
+void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
+void taosCacheDestroyIter(SCacheIter* pIter);
+
 #ifdef __cplusplus
 }
 #endif

include/util/thash.h

@@ -28,11 +28,6 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
 typedef void (*_hash_before_fn_t)(void *);
 typedef void (*_hash_free_fn_t)(void *);
 
-#define HASH_MAX_CAPACITY        (1024 * 1024 * 16)
-#define HASH_DEFAULT_LOAD_FACTOR (0.75)
-
-#define HASH_INDEX(v, c) ((v) & ((c)-1))
-
 #define HASH_NODE_EXIST(code) (code == -2)
 
 /**
@@ -62,41 +57,17 @@ typedef struct SHashNode {
   uint32_t          hashVal;  // the hash value of key
   uint32_t          dataLen;  // length of data
   uint32_t          keyLen;   // length of the key
-  uint16_t          count;    // reference count
+  uint16_t          refCount; // reference count
   int8_t            removed;  // flag to indicate removed
   char              data[];
 } SHashNode;
 
-#define GET_HASH_NODE_KEY(_n)  ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
-#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
-#define GET_HASH_PNODE(_n)     ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
-
 typedef enum SHashLockTypeE {
   HASH_NO_LOCK = 0,
   HASH_ENTRY_LOCK = 1,
 } SHashLockTypeE;
 
-typedef struct SHashEntry {
-  int32_t    num;    // number of elements in current entry
-  SRWLatch   latch;  // entry latch
-  SHashNode *next;
-} SHashEntry;
-
-typedef struct SHashObj {
-  SHashEntry **hashList;
-  uint32_t     capacity;  // number of slots
-  uint32_t     size;      // number of elements in hash table
-
-  _hash_fn_t        hashFp;      // hash function
-  _hash_free_fn_t   freeFp;      // hash node free callback function
-  _equal_fn_t       equalFp;     // equal function
-  _hash_before_fn_t callbackFp;  // function invoked before return the value to caller
-
-  SRWLatch       lock;          // read-write spin lock
-  SHashLockTypeE type;          // lock type
-  bool           enableUpdate;  // enable update
-  SArray        *pMemBlock;     // memory block allocated for SHashEntry
-} SHashObj;
+typedef struct SHashObj SHashObj;
 
 /**
  * init the hash table
@@ -126,8 +97,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
  */
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
 
-int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
-
 /**
  * return the payload data with the specified key
  *
@@ -146,17 +115,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
  * @param destBuf
  * @return
  */
-void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
+int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
 
 /**
- * Clone the result to interval allocated buffer
+ *
  * @param pHashObj
  * @param key
  * @param keyLen
  * @param destBuf
+ * @param size
  * @return
  */
-void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz);
+int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size);
 
 /**
  * remove item with the specified key
@@ -207,37 +177,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
  */
 void taosHashCancelIterate(SHashObj *pHashObj, void *p);
 
-/**
- * Get the corresponding key information for a given data in hash table
- * @param data
- * @return
- */
-int32_t taosHashGetKey(void *data, void **key, size_t *keyLen);
-
-/**
- * Get the corresponding key information for a given data in hash table, using memcpy
- * @param data
- * @param dst
- * @return
- */
-static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) {
-  if (NULL == data || NULL == dst) {
-    return -1;
-  }
-
-  SHashNode *node = GET_HASH_PNODE(data);
-  void      *key = GET_HASH_NODE_KEY(node);
-  memcpy(dst, key, node->keyLen);
-
-  return 0;
-}
-
-/**
- * Get the corresponding data length for a given data in hash table
- * @param data
- * @return
- */
-int32_t taosHashGetDataLen(void *data);
+ /**
+  * Get the corresponding key information for a given data in hash table
+  * @param data
+  * @param keyLen
+  * @return
+  */
+void *taosHashGetKey(void *data, size_t* keyLen);
 
 /**
  * return the payload data with the specified key(reference number added)
@@ -258,8 +204,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
  */
 void taosHashRelease(SHashObj *pHashObj, void *p);
 
+/**
+ *
+ * @param pHashObj
+ * @param fp
+ */
 void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
 
+/**
+ *
+ * @param pHashObj
+ * @param fp
+ */
+void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tpagedbuf.h

@@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis {
  * @param handle
  * @return
  */
-int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
+int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir);
 
 /**
  *

source/client/src/clientHb.c

@@ -473,7 +473,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
     free(pAppHbMgr);
     return NULL;
   }
-  pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
+
+  taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq);
   // init getInfoFunc
   pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
 

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -85,7 +85,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
   int32_t      refCount = 0;
 
   taosRLockLatch(&pMgmt->latch);
-  taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
+  taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
   if (pVnode == NULL) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
   } else {

source/dnode/mnode/impl/src/mndProfile.c

@@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
 static void      mndFreeConn(SConnObj *pConn);
 static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
 static void      mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
-static void     *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn);
+static void     *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
 static void      mndCancelGetNextConn(SMnode *pMnode, void *pIter);
 static int32_t   mndProcessHeartBeatReq(SMnodeMsg *pReq);
 static int32_t   mndProcessConnectReq(SMnodeMsg *pReq);
@@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
   taosCacheRelease(pMgmt->cache, (void **)&pConn, false);
 }
 
-static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) {
-  SProfileMgmt *pMgmt = &pMnode->profileMgmt;
-
-  *pConn = NULL;
-
-  pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter);
-  if (pIter == NULL) return NULL;
-
-  SCacheDataNode **pNode = pIter;
-  if (pNode == NULL || *pNode == NULL) {
-    taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
-    return NULL;
+void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
+  SConnObj* pConn = NULL;
+  bool hasNext = taosCacheIterNext(pIter);
+  if (hasNext) {
+    size_t dataLen = 0;
+    pConn = taosCacheIterGetData(pIter, &dataLen);
+  } else {
+    taosCacheDestroyIter(pIter);
   }
 
-  *pConn = (SConnObj *)((*pNode)->data);
-  return pIter;
+  return pConn;
 }
 
 static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
-  SProfileMgmt *pMgmt = &pMnode->profileMgmt;
-  taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
+  if (pIter != NULL) {
+    taosCacheDestroyIter(pIter);
+  }
 }
 
 static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
@@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
             int32_t rspLen = 0;
             mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
             if (rspMsg && rspLen > 0) {
-              SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
-              taosArrayPush(hbRsp.info, &kv);
+              SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
+              taosArrayPush(hbRsp.info, &kv1);
             }
             break;
           }
@@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
             int32_t rspLen = 0;
             mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
             if (rspMsg && rspLen > 0) {
-              SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
-              taosArrayPush(hbRsp.info, &kv);
+              SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
+              taosArrayPush(hbRsp.info, &kv1);
             }
             break;
           }
@@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable);
+  pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache);
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   strcpy(pMeta->tbName, mndShowStr(pShow->type));
 
@@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
   char     *pWrite;
   char      ipStr[TSDB_IPv4ADDR_LEN + 6];
 
+  if (pShow->pIter == NULL) {
+    SProfileMgmt *pMgmt = &pMnode->profileMgmt;
+    pShow->pIter = taosCacheCreateIter(pMgmt->cache);
+  }
+
   while (numOfRows < rows) {
-    pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
+    pConn = mndGetNextConn(pMnode, pShow->pIter);
     if (pConn == NULL) break;
 
     cols = 0;
@@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
   void     *pIter;
   char      str[TSDB_IPv4ADDR_LEN + 6] = {0};
 
+  if (pShow->pIter == NULL) {
+    SProfileMgmt *pMgmt = &pMnode->profileMgmt;
+    pShow->pIter = taosCacheCreateIter(pMgmt->cache);
+  }
+
   while (numOfRows < rows) {
-    pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
+    pConn = mndGetNextConn(pMnode, pShow->pIter);
     if (pConn == NULL) {
-      pShow->pIter = pIter;
+      pShow->pIter = NULL;
       break;
     }
 
     if (numOfRows + pConn->numOfQueries >= rows) {
-      mndCancelGetNextConn(pMnode, pIter);
+      taosCacheDestroyIter(pShow->pIter);
+      pShow->pIter = NULL;
       break;
     }
 
-    pShow->pIter = pIter;
     for (int32_t i = 0; i < pConn->numOfQueries; ++i) {
       SQueryDesc *pDesc = pConn->pQueries + i;
       cols = 0;
@@ -913,8 +919,9 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
 }
 
 static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
-  SProfileMgmt *pMgmt = &pMnode->profileMgmt;
-  taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
+  if (pIter != NULL) {
+    taosCacheDestroyIter(pIter);
+  }
 }
 
 int32_t mndGetNumOfConnections(SMnode *pMnode) { return taosHashGetSize(pMnode->profileMgmt.cache->pHashTable); }
\ No newline at end of file

source/libs/catalog/src/catalog.c

@@ -168,7 +168,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
     ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
      tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
   }
-  
+
   int32_t colNum = c->numOfColumns + c->numOfTags;
   for (int32_t i = 0; i < colNum; ++i) {
     SSchema *s = &p->schema[i];
@@ -190,7 +190,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
     
     dbCache = (SCtgDBCache *)pIter;
 
-    taosHashGetKey(dbCache, (void **)&dbFName, &len);
+    taosHashGetKey((void **)&dbFName, &len);
 
     int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
     int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
@@ -204,9 +204,9 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
       if (dbCache->vgInfo->vgHash) {
         vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
       }
-    }    
+    }
     
-    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", 
+    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
       i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
 
     pIter = taosHashIterate(dbHash, pIter);
@@ -222,7 +222,7 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) {
   }
 
   ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
-  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 
+  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
     ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
   
   ctgDbgShowDBCache(pCtg, pCtg->dbCache);
@@ -306,9 +306,9 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId)
   return TSDB_CODE_SUCCESS;
 
 _return:
-  
+
   tfree(action.data);
-  CTG_RET(code);  
+  CTG_RET(code);
 }
 
 
@@ -336,9 +336,9 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
   return TSDB_CODE_SUCCESS;
 
 _return:
-  
+
   tfree(action.data);
-  CTG_RET(code);  
+  CTG_RET(code);
 }
 
 
@@ -366,9 +366,9 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
   return TSDB_CODE_SUCCESS;
 
 _return:
-  
+
   tfree(action.data);
-  CTG_RET(code);  
+  CTG_RET(code);
 }
 
 
@@ -657,9 +657,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
     return TSDB_CODE_SUCCESS;
   }
   
-  size_t sz = 0;  
+  int32_t sz = 0;
   CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
-  STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz);
+  int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz);
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
 
   if (NULL == *pTableMeta) {
@@ -673,8 +673,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
   if (dbId) {
     *dbId = dbCache->dbId;
   }
-  
-  tbMeta = *pTableMeta;
+
+  STableMeta* tbMeta = *pTableMeta;
 
   if (tbMeta->tableType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
@@ -1076,7 +1076,7 @@ _return:
 }
 
 int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) {
-  int16_t widx = abs(id % mgmt->slotNum);
+  int16_t widx = labs(id % mgmt->slotNum);
 
   SCtgRentSlot *slot = &mgmt->slots[widx];
   int32_t code = 0;
@@ -1238,7 +1238,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) {
     void *pIter = taosHashIterate(cache->stbCache, NULL);
     while (pIter) {
       uint64_t *suid = NULL;
-      taosHashGetKey(pIter, (void **)&suid, NULL);
+      suid = taosHashGetKey(pIter, NULL);
 
       if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) {
         ctgDebug("stb removed from rent, suid:%"PRIx64, *suid);
@@ -1397,7 +1397,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
         CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
 
         ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
-        
+
         ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare);
       }
 
@@ -1437,7 +1437,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
   if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) {
     CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
     CTG_UNLOCK(CTG_READ, &tbCache->metaLock);    
-    ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid);
+    ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid);
     CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
   }
   
@@ -1475,7 +1475,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
   int32_t *vgId = NULL;
   void *pIter = taosHashIterate(src->vgHash, NULL);
   while (pIter) {
-    taosHashGetKey(pIter, (void **)&vgId, NULL);
+    vgId = taosHashGetKey(pIter, NULL);
 
     if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
       qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
@@ -1635,7 +1635,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
       if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) {
         CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist));
       }
-      
+
       if (0 == exist) {
         CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput));
 
@@ -1723,9 +1723,9 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     tbType = (*pTableMeta)->tableType;
     suid = (*pTableMeta)->suid;
 
-    tfree(*pTableMeta);    
+    tfree(*pTableMeta);
   }
-    
+
   if (CTG_FLAG_IS_UNKNOWN_STB(flag)) {
     CTG_FLAG_SET_STB(flag, tbType);
   }
@@ -1950,21 +1950,21 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
     ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName);
     return TSDB_CODE_SUCCESS;
   }
-  
+
   CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
-  if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {  
+  if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
     CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
     ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
-  }  
+  }
   CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
-  
+
   ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
-  
+
 _return:
 
   tfree(msg);
-  
+
   CTG_RET(code);
 }
 
@@ -2458,7 +2458,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm
     CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList));
   } else {
     int32_t vgId = tbMeta->vgId;
-    if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) {
+    if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) {
       ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName));
       CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);    
     }

source/libs/executor/inc/executil.h

@@ -68,7 +68,7 @@ typedef struct SResultRow {
 } SResultRow;
 
 typedef struct SResultRowInfo {
-  SResultRow  *pCurResult; // current active result row info
+  SList* pRows;
   SResultRow** pResult;    // result list
 //  int16_t      type:8;     // data type for hash key
   int32_t      size;       // number of result set

source/libs/executor/inc/executorimpl.h

@@ -233,9 +233,9 @@ typedef struct STaskAttr {
   SArray*         pUdfInfo;  // no need to free
 } STaskAttr;
 
-typedef int32_t (*__optr_prepare_fn_t)(void* param);
-typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
-typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
+typedef int32_t (*__optr_open_fn_t)(void* param);
+typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup);
+typedef void (*__optr_close_fn_t)(void* param, int32_t num);
 
 struct SOperatorInfo;
 
@@ -306,21 +306,21 @@ enum {
 };
 
 typedef struct SOperatorInfo {
-  uint8_t          operatorType;
-  bool             blockingOptr;  // block operator or not
-  uint8_t          status;        // denote if current operator is completed
-  int32_t          numOfOutput;   // number of columns of the current operator results
-  char*            name;          // name, used to show the query execution plan
-  void*            info;          // extension attribution
-  SExprInfo*       pExpr;
-  STaskRuntimeEnv* pRuntimeEnv;  // todo remove it
-  SExecTaskInfo*   pTaskInfo;
+  uint8_t                operatorType;
+  bool                   blockingOptr;  // block operator or not
+  uint8_t                status;        // denote if current operator is completed
+  int32_t                numOfOutput;   // number of columns of the current operator results
+  char*                  name;          // name, used to show the query execution plan
+  void*                  info;          // extension attribution
+  SExprInfo*             pExpr;
+  STaskRuntimeEnv*       pRuntimeEnv;  // todo remove it
+  SExecTaskInfo*         pTaskInfo;
 
   struct SOperatorInfo** pDownstream;      // downstram pointer list
   int32_t                numOfDownstream;  // number of downstream. The value is always ONE expect for join operator
-  __optr_prepare_fn_t    prepareFn;
-  __operator_fn_t        exec;
-  __optr_cleanup_fn_t    cleanupFn;
+  __optr_open_fn_t       openFn;
+  __optr_fn_t            nextDataFn;
+  __optr_close_fn_t      closeFn;
 } SOperatorInfo;
 
 typedef struct {
@@ -479,9 +479,6 @@ typedef struct SAggOperatorInfo {
 
 typedef struct SProjectOperatorInfo {
   SOptrBasicInfo binfo;
-  int32_t        bufCapacity;
-  uint32_t       seed;
-
   SSDataBlock* existDataBlock;
 } SProjectOperatorInfo;
 
@@ -615,10 +612,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
-SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
-                                         int32_t numOfOutput);
+SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
+
 SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
-SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
                                                  SExprInfo* pExpr, int32_t numOfOutput);
@@ -654,8 +651,6 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf
 SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
 
-// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
-// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
 // SSDataBlock* doSLimit(void* param, bool* newgroup);
 
 // int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);

source/libs/executor/inc/tsimplehash.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TSIMPLEHASH_H
+#define TDENGINE_TSIMPLEHASH_H
+
+#include "tarray.h"
+#include "tlockfree.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
+typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
+typedef void (*_hash_free_fn_t)(void *);
+
+typedef struct SSHashObj SSHashObj;
+
+/**
+ * init the hash table
+ *
+ * @param capacity    initial capacity of the hash table
+ * @param fn          hash function to generate the hash value
+ * @return
+ */
+SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen);
+
+/**
+ * return the size of hash table
+ * @param pHashObj
+ * @return
+ */
+int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
+
+/**
+ * put element into hash table, if the element with the same key exists, update it
+ * @param pHashObj
+ * @param key
+ * @param data
+ * @return
+ */
+int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data);
+
+/**
+ * return the payload data with the specified key
+ *
+ * @param pHashObj
+ * @param key
+ * @return
+ */
+void *tSimpleHashGet(SSHashObj *pHashObj, const void *key);
+
+/**
+ * remove item with the specified key
+ * @param pHashObj
+ * @param key
+ * @param keyLen
+ */
+int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key);
+
+/**
+ * Clear the hash table.
+ * @param pHashObj
+ */
+void tSimpleHashClear(SSHashObj *pHashObj);
+
+/**
+ * Clean up hash table and release all allocated resources.
+ * @param handle
+ */
+void tSimpleHashCleanup(SSHashObj *pHashObj);
+
+/**
+ * Get the hash table size
+ * @param pHashObj
+ * @return
+ */
+size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj);
+
+/**
+ * Get the corresponding key information for a given data in hash table
+ * @param data
+ * @param keyLen
+ * @return
+ */
+void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen);
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // TDENGINE_TSIMPLEHASH_H

source/libs/executor/src/executorMain.c

@@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
   int64_t st = 0;
 
   st = taosGetTimestampUs();
-  *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup);
+  *pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup);
 
   uint64_t el = (taosGetTimestampUs() - st);
   pTaskInfo->cost.elapsedTime += el;

source/libs/executor/src/executorimpl.c

@@ -4619,7 +4619,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
   getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize);
 
   int32_t TENMB = 1024*1024*10;
-  int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp");
+  int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp");
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -5281,7 +5281,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
   pOperator->status       = OP_IN_EXECUTING;
   pOperator->info         = pInfo;
   pOperator->numOfOutput  = size;
-  pOperator->exec         = doLoadRemoteData;
+  pOperator->nextDataFn = doLoadRemoteData;
   pOperator->pTaskInfo    = pTaskInfo;
 
 #if 1
@@ -5361,7 +5361,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
   pOperator->status        = OP_IN_EXECUTING;
   pOperator->info          = pInfo;
   pOperator->numOfOutput   = numOfOutput;
-  pOperator->exec          = doTableScan;
+  pOperator->nextDataFn = doTableScan;
   pOperator->pTaskInfo     = pTaskInfo;
 
   return pOperator;
@@ -5386,7 +5386,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
   pOperator->info         = pInfo;
   pOperator->numOfOutput  = pRuntimeEnv->pQueryAttr->numOfCols;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = doTableScanImpl;
+  pOperator->nextDataFn = doTableScanImpl;
 
   return pOperator;
 }
@@ -5410,7 +5410,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt
   pOperator->status       = OP_IN_EXECUTING;
   pOperator->info         = pInfo;
 //  pOperator->numOfOutput  = pRuntimeEnv->pQueryAttr->numOfCols;
-  pOperator->exec         = doBlockInfoScan;
+  pOperator->nextDataFn = doBlockInfoScan;
 
   return pOperator;
 }
@@ -5452,7 +5452,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp
   pOperator->status        = OP_IN_EXECUTING;
   pOperator->info          = pInfo;
   pOperator->numOfOutput   = numOfOutput;
-  pOperator->exec          = doStreamBlockScan;
+  pOperator->nextDataFn = doStreamBlockScan;
   pOperator->pTaskInfo     = pTaskInfo;
   return pOperator;
 }
@@ -5663,7 +5663,7 @@ SSDataBlock* loadNextDataBlock(void* param) {
   SOperatorInfo* pOperator = (SOperatorInfo*) param;
   bool newgroup = false;
 
-  return pOperator->exec(pOperator, &newgroup);
+  return pOperator->nextDataFn(pOperator, &newgroup);
 }
 
 static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) {
@@ -5983,8 +5983,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t
   pOperator->pExpr        = exprArrayDup(pExprInfo);
 
   pOperator->pTaskInfo    = pTaskInfo;
-  pOperator->exec         = doSortedMerge;
-  pOperator->cleanupFn    = destroySortedMergeOperatorInfo;
+  pOperator->nextDataFn = doSortedMerge;
+  pOperator->closeFn = destroySortedMergeOperatorInfo;
 
   code = appendDownstream(pOperator, downstream, numOfDownstream);
   if (code != TSDB_CODE_SUCCESS) {
@@ -6079,8 +6079,8 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI
   pOperator->info          = pInfo;
 
   pOperator->pTaskInfo     = pTaskInfo;
-  pOperator->exec          = doSort;
-  pOperator->cleanupFn     = destroyOrderOperatorInfo;
+  pOperator->nextDataFn = doSort;
+  pOperator->closeFn = destroyOrderOperatorInfo;
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -6105,7 +6105,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6155,7 +6155,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6224,7 +6224,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
 
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
-    updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
+    updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
 
     projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
 
@@ -6241,7 +6241,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
 
     // The downstream exec may change the value of the newgroup, so use a local variable instead.
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
+    SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6274,7 +6274,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
 
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
-    updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
+    updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
 
     projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
     pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
@@ -6299,7 +6299,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) {
   SSDataBlock* pBlock = NULL;
   while (1) {
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
+    pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6350,7 +6350,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) {
 
   while (1) {
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
+    SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6393,7 +6393,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6453,7 +6453,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6516,7 +6516,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6571,7 +6571,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6706,7 +6706,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
   SOperatorInfo* downstream = pOperator->pDownstream[0];
   while (1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -6768,7 +6768,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
     if (pBlock == NULL) {
       break;
@@ -6821,7 +6821,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
+    SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
     publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
     if (pBlock == NULL) {
       break;
@@ -6906,7 +6906,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) {
 
   while(1) {
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
+    SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (*newgroup) {
@@ -6979,8 +6979,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
     return;
   }
 
-  if (pOperator->cleanupFn != NULL) {
-    pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput);
+  if (pOperator->closeFn != NULL) {
+    pOperator->closeFn(pOperator->info, pOperator->numOfOutput);
   }
 
   if (pOperator->pDownstream != NULL) {
@@ -7067,8 +7067,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE
   pOperator->numOfOutput  = taosArrayGetSize(pExprInfo);
 
   pOperator->pTaskInfo    = pTaskInfo;
-  pOperator->exec         = doAggregate;
-  pOperator->cleanupFn    = destroyAggOperatorInfo;
+  pOperator->nextDataFn = doAggregate;
+  pOperator->closeFn = destroyAggOperatorInfo;
   int32_t code = appendDownstream(pOperator, &downstream, 1);
 
   return pOperator;
@@ -7164,38 +7164,34 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray
   pOperator->pExpr        = exprArrayDup(pExprInfo);
   pOperator->numOfOutput  = numOfOutput;
 
-  pOperator->exec         = doMultiTableAggregate;
-  pOperator->cleanupFn    = destroyAggOperatorInfo;
+  pOperator->nextDataFn = doMultiTableAggregate;
+  pOperator->closeFn = destroyAggOperatorInfo;
   int32_t code = appendDownstream(pOperator, &downstream, 1);
 
   return pOperator;
 }
 
-SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
+SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
   SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo));
 
-  pInfo->seed = rand();
-  pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity;
-
-  SOptrBasicInfo* pBInfo = &pInfo->binfo;
-  pBInfo->pRes  = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity);
-  pBInfo->pCtx  = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
+  int32_t numOfRows = 4096;
+  pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows);
+  pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
 
-  initResultRowInfo(&pBInfo->resultRowInfo, 8);
-  setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN);
+//  initResultRowInfo(&pBInfo->resultRowInfo, 8);
+//  setDefaultOutputBuf_rv(pBInfo, MAIN_SCAN);
 
   SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
   pOperator->name         = "ProjectOperator";
-//  pOperator->operatorType = OP_Project;
+  pOperator->operatorType = OP_Project;
   pOperator->blockingOptr = false;
   pOperator->status       = OP_IN_EXECUTING;
   pOperator->info         = pInfo;
-  pOperator->pExpr        = pExpr;
-  pOperator->numOfOutput  = numOfOutput;
-  pOperator->pRuntimeEnv  = pRuntimeEnv;
+  pOperator->pExpr        = exprArrayDup(pExprInfo);
+  pOperator->numOfOutput  = taosArrayGetSize(pExprInfo);
 
-  pOperator->exec         = doProjectOperation;
-  pOperator->cleanupFn    = destroyProjectOperatorInfo;
+  pOperator->nextDataFn   = doProjectOperation;
+  pOperator->closeFn      = destroyProjectOperatorInfo;
   int32_t code = appendDownstream(pOperator, &downstream, 1);
 
   return pOperator;
@@ -7250,10 +7246,10 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
   pOperator->status       = OP_IN_EXECUTING;
   pOperator->numOfOutput  = numOfOutput;
   pOperator->pExpr        = pExpr;
-  pOperator->exec         = doFilter;
+  pOperator->nextDataFn = doFilter;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->cleanupFn    = destroyConditionOperatorInfo;
+  pOperator->closeFn = destroyConditionOperatorInfo;
   int32_t code = appendDownstream(pOperator, &downstream, 1);
 
   return pOperator;
@@ -7269,7 +7265,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
 //  pOperator->operatorType = OP_Limit;
   pOperator->blockingOptr = false;
   pOperator->status       = OP_IN_EXECUTING;
-  pOperator->exec         = doLimit;
+  pOperator->nextDataFn = doLimit;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
     int32_t code = appendDownstream(pOperator, &downstream, 1);
@@ -7277,24 +7273,18 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
   return pOperator;
 }
 
-SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) {
   STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
 
   initAggSup(&pInfo->aggSup, pExprInfo);
 
-  // todo:
   pInfo->order = TSDB_ORDER_ASC;
   pInfo->precision = TSDB_TIME_PRECISION_MICRO;
-  pInfo->win.skey = INT64_MIN;
-  pInfo->win.ekey = INT64_MAX;
-  pInfo->interval.intervalUnit = 's';
-  pInfo->interval.slidingUnit  = 's';
-  pInfo->interval.interval = 1000;
-  pInfo->interval.sliding  = 1000;
+  pInfo->win   = pTaskInfo->window;
+  pInfo->interval = *pInterval;
 
-  int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/");
+  int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/");
 
-  int32_t numOfOutput = taosArrayGetSize(pExprInfo);
   pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
   pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity);
 
@@ -7309,16 +7299,15 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx
   pOperator->pExpr        = exprArrayDup(pExprInfo);
 
   pOperator->pTaskInfo    = pTaskInfo;
-  pOperator->numOfOutput  = numOfOutput;
+  pOperator->numOfOutput  = taosArrayGetSize(pExprInfo);
   pOperator->info         = pInfo;
-  pOperator->exec         = doIntervalAgg;
-  pOperator->cleanupFn    = destroyBasicOperatorInfo;
+  pOperator->nextDataFn   = doIntervalAgg;
+  pOperator->closeFn      = destroyBasicOperatorInfo;
 
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 }
 
-
 SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
   STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
 
@@ -7336,8 +7325,8 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = doAllIntervalAgg;
-  pOperator->cleanupFn    = destroyBasicOperatorInfo;
+  pOperator->nextDataFn = doAllIntervalAgg;
+  pOperator->closeFn = destroyBasicOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7360,8 +7349,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = doStateWindowAgg;
-  pOperator->cleanupFn    = destroyStateWindowOperatorInfo;
+  pOperator->nextDataFn = doStateWindowAgg;
+  pOperator->closeFn = destroyStateWindowOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7385,8 +7374,8 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = doSessionWindowAgg;
-  pOperator->cleanupFn    = destroySWindowOperatorInfo;
+  pOperator->nextDataFn = doSessionWindowAgg;
+  pOperator->closeFn = destroySWindowOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7409,8 +7398,8 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
 
-  pOperator->exec         = doSTableIntervalAgg;
-  pOperator->cleanupFn    = destroyBasicOperatorInfo;
+  pOperator->nextDataFn = doSTableIntervalAgg;
+  pOperator->closeFn = destroyBasicOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7433,8 +7422,8 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
 
-  pOperator->exec         = doAllSTableIntervalAgg;
-  pOperator->cleanupFn    = destroyBasicOperatorInfo;
+  pOperator->nextDataFn = doAllSTableIntervalAgg;
+  pOperator->closeFn = destroyBasicOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
 
@@ -7465,8 +7454,8 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = hashGroupbyAggregate;
-  pOperator->cleanupFn    = destroyGroupbyOperatorInfo;
+  pOperator->nextDataFn = hashGroupbyAggregate;
+  pOperator->closeFn = destroyGroupbyOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7504,8 +7493,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = doFill;
-  pOperator->cleanupFn    = destroySFillOperatorInfo;
+  pOperator->nextDataFn = doFill;
+  pOperator->closeFn = destroySFillOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7553,7 +7542,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
 //  pOperator->exec         = doSLimit;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->cleanupFn    = destroySlimitOperatorInfo;
+  pOperator->closeFn = destroySlimitOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
@@ -7707,11 +7696,11 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo
   pOperator->blockingOptr = false;
   pOperator->status       = OP_IN_EXECUTING;
   pOperator->info         = pInfo;
-  pOperator->exec         = doTagScan;
+  pOperator->nextDataFn = doTagScan;
   pOperator->pExpr        = pExpr;
   pOperator->numOfOutput  = numOfOutput;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->cleanupFn    = destroyTagScanOperatorInfo;
+  pOperator->closeFn = destroyTagScanOperatorInfo;
 
   return pOperator;
 }
@@ -7777,7 +7766,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
    
   while(1) {
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
-    pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
+    pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
     publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
 
     if (pBlock == NULL) {
@@ -7849,9 +7838,9 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato
   pOperator->numOfOutput  = numOfOutput;
   pOperator->info         = pInfo;
   pOperator->pRuntimeEnv  = pRuntimeEnv;
-  pOperator->exec         = hashDistinct;
+  pOperator->nextDataFn = hashDistinct;
   pOperator->pExpr        = pExpr; 
-  pOperator->cleanupFn    = destroyDistinctOperatorInfo;
+  pOperator->closeFn = destroyDistinctOperatorInfo;
 
     int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;

source/libs/executor/src/tlinearhash.c

@@ -29,8 +29,8 @@ typedef struct SLHashBucket {
 typedef struct SLHashObj {
   SDiskbasedBuf *pBuf;
   _hash_fn_t     hashFn;
-  int32_t        tuplesPerPage;
   SLHashBucket **pBucket;         // entry list
+  int32_t        tuplesPerPage;
   int32_t        numOfAlloc;      // number of allocated bucket ptr slot
   int32_t        bits;            // the number of bits used in hash
   int32_t        numOfBuckets;    // the number of buckets
@@ -142,7 +142,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket
   pBucket->size -= 1;
 }
 
-static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
+static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
   size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList);
   if (numOfPages <= 1) {
     return;
@@ -253,6 +253,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
     return NULL;
   }
 
+  // disable compress when flushing to disk
   setBufPageCompressOnDisk(pHashObj->pBuf, false);
 
   /**
@@ -367,7 +368,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
       releaseBufPage(pHashObj->pBuf, p);
     }
 
-    doCompressBucketPages(pHashObj, pBucket);
+    doTrimBucketPages(pHashObj, pBucket);
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/executor/src/tsimplehash.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tsimplehash.h"
+#include "taoserror.h"
+
+#define SHASH_DEFAULT_LOAD_FACTOR 0.75
+#define HASH_MAX_CAPACITY         (1024*1024*16)
+#define SHASH_NEED_RESIZE(_h)     ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
+
+#define GET_SHASH_NODE_KEY(_n, _dl)  ((char*)(_n) + sizeof(SHNode) + (_dl))
+#define GET_SHASH_NODE_DATA(_n)      ((char*)(_n) + sizeof(SHNode))
+
+#define HASH_INDEX(v, c)         ((v) & ((c)-1))
+#define HASH_NEED_RESIZE(_h)     ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
+
+#define FREE_HASH_NODE(_n) \
+  do {                     \
+    tfree(_n);             \
+  } while (0);
+
+typedef struct SHNode {
+  struct SHNode   *next;
+  char             data[];
+} SHNode;
+
+typedef struct SSHashObj {
+  SHNode         **hashList;
+  size_t           capacity;     // number of slots
+  size_t           size;         // number of elements in hash table
+  _hash_fn_t       hashFp;       // hash function
+  _equal_fn_t      equalFp;      // equal function
+  int32_t          keyLen;
+  int32_t          dataLen;
+} SSHashObj;
+
+static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
+  int32_t len = MIN(length, HASH_MAX_CAPACITY);
+
+  int32_t i = 4;
+  while (i < len) i = (i << 1u);
+  return i;
+}
+
+SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) {
+  ASSERT(fn != NULL);
+
+  if (capacity == 0) {
+    capacity = 4;
+  }
+
+  SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj));
+  if (pHashObj == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  // the max slots is not defined by user
+  pHashObj->capacity = taosHashCapacity((int32_t)capacity);
+
+  pHashObj->equalFp = memcmp;
+  pHashObj->hashFp  = fn;
+  ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
+
+  pHashObj->keyLen = keyLen;
+  pHashObj->dataLen = dataLen;
+
+  pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *));
+  if (pHashObj->hashList == NULL) {
+    free(pHashObj);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  return pHashObj;
+}
+
+int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) {
+  if (pHashObj == NULL) {
+    return 0;
+  }
+  return (int32_t)atomic_load_64(&pHashObj->size);
+}
+
+static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
+  SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize);
+  if (pNewNode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pNewNode->next = NULL;
+  memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize);
+  memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen);
+  return pNewNode;
+}
+
+void taosHashTableResize(SSHashObj *pHashObj) {
+  if (!HASH_NEED_RESIZE(pHashObj)) {
+    return;
+  }
+
+  int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
+  if (newCapacity > HASH_MAX_CAPACITY) {
+//    uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
+//           pHashObj->capacity, HASH_MAX_CAPACITY);
+    return;
+  }
+
+  int64_t st = taosGetTimestampUs();
+  void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity);
+  if (pNewEntryList == NULL) {
+//    qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
+    return;
+  }
+
+  size_t inc = newCapacity - pHashObj->capacity;
+  memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc);
+
+  pHashObj->hashList = pNewEntryList;
+  pHashObj->capacity = newCapacity;
+
+  for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
+    SHNode* pNode = pHashObj->hashList[idx];
+    SHNode *pNext;
+    SHNode *pPrev = NULL;
+
+    if (pNode == NULL) {
+      continue;
+    }
+
+    while (pNode != NULL) {
+      void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen);
+      uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen);
+
+      int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity);
+      pNext = pNode->next;
+      if (newIdx != idx) {
+        if (pPrev == NULL) {
+          pHashObj->hashList[idx] = pNext;
+        } else {
+          pPrev->next = pNext;
+        }
+
+        pNode->next = pHashObj->hashList[newIdx];
+        pHashObj->hashList[newIdx] = pNode;
+      } else {
+        pPrev = pNode;
+      }
+
+      pNode = pNext;
+    }
+  }
+
+  int64_t et = taosGetTimestampUs();
+
+//  uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
+//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
+}
+
+int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
+  if (pHashObj == NULL || key == NULL) {
+    return -1;
+  }
+
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+
+  // need the resize process, write lock applied
+  if (SHASH_NEED_RESIZE(pHashObj)) {
+    taosHashTableResize(pHashObj);
+  }
+
+  int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
+
+  SHNode *pNode = pHashObj->hashList[slot];
+  if (pNode == NULL) {
+    SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
+    if (pNewNode == NULL) {
+      return -1;
+    }
+
+    pHashObj->hashList[slot] = pNewNode;
+    return 0;
+  }
+
+  while (pNode) {
+    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
+      break;
+    }
+    pNode = pNode->next;
+  }
+
+  if (pNode == NULL) {
+    SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
+    if (pNewNode == NULL) {
+      return -1;
+    }
+    pNewNode->next = pHashObj->hashList[slot];
+    pHashObj->hashList[slot] = pNewNode;
+    atomic_add_fetch_64(&pHashObj->size, 1);
+  } else {  //update data
+    memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen);
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) {
+  SHNode *pNode = pHashObj->hashList[index];
+  while (pNode) {
+    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
+      break;
+    }
+
+    pNode = pNode->next;
+  }
+
+  return pNode;
+}
+
+static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) {
+  return tSimpleHashGetSize(pHashObj) == 0;
+}
+
+void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
+  if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) {
+    return NULL;
+  }
+
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+
+  int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
+  SHNode *pNode = pHashObj->hashList[slot];
+  if (pNode == NULL) {
+    return NULL;
+  }
+
+  char *data = NULL;
+  pNode = doSearchInEntryList(pHashObj, key, slot);
+  if (pNode != NULL) {
+    data = GET_SHASH_NODE_DATA(pNode);
+  }
+
+  return data;
+}
+
+int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) {
+  // todo
+}
+
+void tSimpleHashClear(SSHashObj *pHashObj) {
+  if (pHashObj == NULL) {
+    return;
+  }
+
+  SHNode *pNode, *pNext;
+  for (int32_t i = 0; i < pHashObj->capacity; ++i) {
+    pNode = pHashObj->hashList[i];
+    if (pNode == NULL) {
+      continue;
+    }
+
+    while (pNode) {
+      pNext = pNode->next;
+      FREE_HASH_NODE(pNode);
+      pNode = pNext;
+    }
+  }
+  pHashObj->size = 0;
+}
+
+void tSimpleHashCleanup(SSHashObj *pHashObj) {
+  if (pHashObj == NULL) {
+    return;
+  }
+
+  tSimpleHashClear(pHashObj);
+  tfree(pHashObj->hashList);
+}
+
+size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) {
+  if (pHashObj == NULL) {
+    return 0;
+  }
+
+  return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj);
+}
+
+void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) {
+  int32_t offset = offsetof(SHNode, data);
+  SHNode *node = data - offset;
+  if (keyLen != NULL) {
+    *keyLen = pHashObj->keyLen;
+  }
+
+  return GET_SHASH_NODE_KEY(node, pHashObj->dataLen);
+}
\ No newline at end of file

source/libs/executor/test/executorTests.cpp

@@ -201,9 +201,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_
   pOperator->name = "dummyInputOpertor4Test";
 
   if (numOfCols == 1) {
-    pOperator->exec = getDummyBlock;
+    pOperator->nextDataFn = getDummyBlock;
   } else {
-    pOperator->exec = get2ColsDummyBlock;
+    pOperator->nextDataFn = get2ColsDummyBlock;
   }
 
   SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
@@ -222,100 +222,600 @@ int main(int argc, char** argv) {
   return RUN_ALL_TESTS();
 }
 
-#if 0
 TEST(testCase, build_executor_tree_Test) {
   const char* msg = "{\n"
-                    "\t\"Id\":\t{\n"
-                    "\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
-                    "\t\t\"TemplateId\":\t0,\n"
-                    "\t\t\"SubplanId\":\t0\n"
-                    "\t},\n"
-                    "\t\"Node\":\t{\n"
-                    "\t\t\"Name\":\t\"TableScan\",\n"
-                    "\t\t\"Targets\":\t[{\n"
-                    "\t\t\t\t\"Base\":\t{\n"
-                    "\t\t\t\t\t\"Schema\":\t{\n"
-                    "\t\t\t\t\t\t\"Type\":\t9,\n"
-                    "\t\t\t\t\t\t\"ColId\":\t5000,\n"
-                    "\t\t\t\t\t\t\"Bytes\":\t8\n"
-                    "\t\t\t\t\t},\n"
-                    "\t\t\t\t\t\"Columns\":\t[{\n"
-                    "\t\t\t\t\t\t\t\"TableId\":\t1,\n"
-                    "\t\t\t\t\t\t\t\"Flag\":\t0,\n"
-                    "\t\t\t\t\t\t\t\"Info\":\t{\n"
-                    "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
-                    "\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
-                    "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
-                    "\t\t\t\t\t\t\t}\n"
-                    "\t\t\t\t\t\t}],\n"
-                    "\t\t\t\t\t\"InterBytes\":\t0\n"
-                    "\t\t\t\t},\n"
-                    "\t\t\t\t\"Expr\":\t{\n"
-                    "\t\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\t\"Column\":\t{\n"
-                    "\t\t\t\t\t\t\"Type\":\t9,\n"
-                    "\t\t\t\t\t\t\"ColId\":\t1,\n"
-                    "\t\t\t\t\t\t\"Bytes\":\t8\n"
-                    "\t\t\t\t\t}\n"
-                    "\t\t\t\t}\n"
-                    "\t\t\t}, {\n"
-                    "\t\t\t\t\"Base\":\t{\n"
-                    "\t\t\t\t\t\"Schema\":\t{\n"
-                    "\t\t\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\t\t\"ColId\":\t5001,\n"
-                    "\t\t\t\t\t\t\"Bytes\":\t4\n"
-                    "\t\t\t\t\t},\n"
-                    "\t\t\t\t\t\"Columns\":\t[{\n"
-                    "\t\t\t\t\t\t\t\"TableId\":\t1,\n"
-                    "\t\t\t\t\t\t\t\"Flag\":\t0,\n"
-                    "\t\t\t\t\t\t\t\"Info\":\t{\n"
-                    "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
-                    "\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
-                    "\t\t\t\t\t\t\t}\n"
-                    "\t\t\t\t\t\t}],\n"
-                    "\t\t\t\t\t\"InterBytes\":\t0\n"
-                    "\t\t\t\t},\n"
-                    "\t\t\t\t\"Expr\":\t{\n"
-                    "\t\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\t\"Column\":\t{\n"
-                    "\t\t\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\t\t\"ColId\":\t2,\n"
-                    "\t\t\t\t\t\t\"Bytes\":\t4\n"
-                    "\t\t\t\t\t}\n"
-                    "\t\t\t\t}\n"
-                    "\t\t\t}],\n"
-                    "\t\t\"InputSchema\":\t[{\n"
-                    "\t\t\t\t\"Type\":\t9,\n"
-                    "\t\t\t\t\"ColId\":\t5000,\n"
-                    "\t\t\t\t\"Bytes\":\t8\n"
-                    "\t\t\t}, {\n"
-                    "\t\t\t\t\"Type\":\t4,\n"
-                    "\t\t\t\t\"ColId\":\t5001,\n"
-                    "\t\t\t\t\"Bytes\":\t4\n"
-                    "\t\t\t}],\n"
-                    "\t\t\"TableScan\":\t{\n"
-                    "\t\t\t\"TableId\":\t1,\n"
-                    "\t\t\t\"TableType\":\t2,\n"
-                    "\t\t\t\"Flag\":\t0,\n"
-                    "\t\t\t\"Window\":\t{\n"
-                    "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
-                    "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
-                    "\t\t\t}\n"
-                    "\t\t}\n"
-                    "\t},\n"
-                    "\t\"DataSink\":\t{\n"
-                    "\t\t\"Name\":\t\"Dispatch\",\n"
-                    "\t\t\"Dispatch\":\t{\n"
-                    "\t\t}\n"
-                    "\t}\n"
+                    "    \"Type\": \"33\",\n"
+                    "    \"Name\": \"PhysiProject\",\n"
+                    "    \"PhysiProject\": {\n"
+                    "        \"OutputDataBlockDesc\": {\n"
+                    "            \"Type\": \"19\",\n"
+                    "            \"Name\": \"TupleDesc\",\n"
+                    "            \"TupleDesc\": {\n"
+                    "                \"DataBlockId\": \"1\",\n"
+                    "                \"Slots\": [\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"0\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"9\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"1\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"4\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"4\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"2\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"8\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"20\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"3\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"5\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"4\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"7\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    {\n"
+                    "                        \"Type\": \"20\",\n"
+                    "                        \"Name\": \"SlotDesc\",\n"
+                    "                        \"SlotDesc\": {\n"
+                    "                            \"SlotId\": \"5\",\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"7\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"Reserve\": false,\n"
+                    "                            \"Output\": false\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                ]\n"
+                    "            }\n"
+                    "        },\n"
+                    "        \"Children\": [\n"
+                    "            {\n"
+                    "                \"Type\": \"30\",\n"
+                    "                \"Name\": \"PhysiTableScan\",\n"
+                    "                \"PhysiTableScan\": {\n"
+                    "                    \"OutputDataBlockDesc\": {\n"
+                    "                        \"Type\": \"19\",\n"
+                    "                        \"Name\": \"TupleDesc\",\n"
+                    "                        \"TupleDesc\": {\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"Slots\": [\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"0\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"9\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                },\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"1\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"4\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"4\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                },\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"2\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"8\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"20\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                },\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"3\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"5\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                },\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"4\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"7\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                },\n"
+                    "                                {\n"
+                    "                                    \"Type\": \"20\",\n"
+                    "                                    \"Name\": \"SlotDesc\",\n"
+                    "                                    \"SlotDesc\": {\n"
+                    "                                        \"SlotId\": \"5\",\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"7\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"Reserve\": false,\n"
+                    "                                        \"Output\": true\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            ]\n"
+                    "                        }\n"
+                    "                    },\n"
+                    "                    \"ScanCols\": [\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"0\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"9\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"ts\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"1\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"ts\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        },\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"1\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"4\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"4\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"c1\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"2\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"c1\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        },\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"2\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"8\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"20\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"c2\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"3\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"c2\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        },\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"3\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"5\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"c3\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"4\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"c3\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        },\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"4\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"7\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"c4\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"5\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"c4\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        },\n"
+                    "                        {\n"
+                    "                            \"Type\": \"18\",\n"
+                    "                            \"Name\": \"Target\",\n"
+                    "                            \"Target\": {\n"
+                    "                                \"DataBlockId\": \"0\",\n"
+                    "                                \"SlotId\": \"5\",\n"
+                    "                                \"Expr\": {\n"
+                    "                                    \"Type\": \"1\",\n"
+                    "                                    \"Name\": \"Column\",\n"
+                    "                                    \"Column\": {\n"
+                    "                                        \"DataType\": {\n"
+                    "                                            \"Type\": \"7\",\n"
+                    "                                            \"Precision\": \"0\",\n"
+                    "                                            \"Scale\": \"0\",\n"
+                    "                                            \"Bytes\": \"8\"\n"
+                    "                                        },\n"
+                    "                                        \"AliasName\": \"c5\",\n"
+                    "                                        \"TableId\": \"0\",\n"
+                    "                                        \"ColId\": \"6\",\n"
+                    "                                        \"ColType\": \"1\",\n"
+                    "                                        \"DbName\": \"test\",\n"
+                    "                                        \"TableName\": \"t1\",\n"
+                    "                                        \"TableAlias\": \"t1\",\n"
+                    "                                        \"ColName\": \"c5\",\n"
+                    "                                        \"DataBlockId\": \"0\",\n"
+                    "                                        \"SlotId\": \"0\"\n"
+                    "                                    }\n"
+                    "                                }\n"
+                    "                            }\n"
+                    "                        }\n"
+                    "                    ],\n"
+                    "                    \"TableId\": \"1\",\n"
+                    "                    \"TableType\": \"3\",\n"
+                    "                    \"ScanOrder\": \"1\",\n"
+                    "                    \"ScanCount\": \"1\",\n"
+                    "                    \"ReverseScanCount\": \"0\",\n"
+                    "                    \"ScanFlag\": \"0\",\n"
+                    "                    \"StartKey\": \"-9223372036854775808\",\n"
+                    "                    \"EndKey\": \"9223372036854775807\"\n"
+                    "                }\n"
+                    "            }\n"
+                    "        ],\n"
+                    "        \"Projections\": [\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"0\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"9\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"ts\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"1\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"ts\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"0\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            },\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"1\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"4\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"4\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"c1\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"2\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"c1\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"1\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            },\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"2\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"8\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"20\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"c2\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"3\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"c2\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"2\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            },\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"3\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"5\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"c3\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"4\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"c3\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"3\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            },\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"4\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"7\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"c4\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"5\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"c4\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"4\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            },\n"
+                    "            {\n"
+                    "                \"Type\": \"18\",\n"
+                    "                \"Name\": \"Target\",\n"
+                    "                \"Target\": {\n"
+                    "                    \"DataBlockId\": \"1\",\n"
+                    "                    \"SlotId\": \"5\",\n"
+                    "                    \"Expr\": {\n"
+                    "                        \"Type\": \"1\",\n"
+                    "                        \"Name\": \"Column\",\n"
+                    "                        \"Column\": {\n"
+                    "                            \"DataType\": {\n"
+                    "                                \"Type\": \"7\",\n"
+                    "                                \"Precision\": \"0\",\n"
+                    "                                \"Scale\": \"0\",\n"
+                    "                                \"Bytes\": \"8\"\n"
+                    "                            },\n"
+                    "                            \"AliasName\": \"c5\",\n"
+                    "                            \"TableId\": \"0\",\n"
+                    "                            \"ColId\": \"6\",\n"
+                    "                            \"ColType\": \"1\",\n"
+                    "                            \"DbName\": \"test\",\n"
+                    "                            \"TableName\": \"t1\",\n"
+                    "                            \"TableAlias\": \"t1\",\n"
+                    "                            \"ColName\": \"c5\",\n"
+                    "                            \"DataBlockId\": \"0\",\n"
+                    "                            \"SlotId\": \"5\"\n"
+                    "                        }\n"
+                    "                    }\n"
+                    "                }\n"
+                    "            }\n"
+                    "        ]\n"
+                    "    }\n"
                     "}";
 
   SExecTaskInfo* pTaskInfo = nullptr;
   DataSinkHandle sinkHandle = nullptr;
   SReadHandle handle = {.reader = reinterpret_cast<void*>(0x1), .meta = reinterpret_cast<void*>(0x1)};
 
-//  int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
+  SSubplan* plan = NULL;
+  qStringToSubplan(msg, &plan);
+
+  int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
 }
 
 TEST(testCase, inMem_sort_Test) {
@@ -334,10 +834,10 @@ TEST(testCase, inMem_sort_Test) {
   exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
   taosArrayPush(pExprInfo, &exp1);
 
-  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL);
+  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
 
   bool newgroup = false;
-  SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup);
+  SSDataBlock* pRes = pOperator->nextDataFn(pOperator, &newgroup);
 
   SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
   SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
@@ -400,7 +900,7 @@ TEST(testCase, external_sort_Test) {
   exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
 //  taosArrayPush(pExprInfo, &exp1);
 
-  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL);
+  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
 
   bool newgroup = false;
   SSDataBlock* pRes = NULL;
@@ -412,7 +912,7 @@ TEST(testCase, external_sort_Test) {
 
   while(1) {
     int64_t s = taosGetTimestampUs();
-    pRes = pOperator->exec(pOperator, &newgroup);
+    pRes = pOperator->nextDataFn(pOperator, &newgroup);
 
     int64_t e = taosGetTimestampUs();
     if (t++ == 1) {
@@ -435,7 +935,7 @@ TEST(testCase, external_sort_Test) {
   int64_t s2 = taosGetTimestampUs();
   printf("total:%ld\n", s2 - s1);
 
-  pOperator->cleanupFn(pOperator->info, 2);
+  pOperator->closeFn(pOperator->info, 2);
   tfree(exp);
   tfree(exp1);
   taosArrayDestroy(pExprInfo);
@@ -469,7 +969,7 @@ TEST(testCase, sorted_merge_Test) {
   int32_t numOfSources = 10;
   SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*));
   for(int32_t i = 0; i < numOfSources; ++i) {
-    plist[i] = createDummyOperator(1, 1, 1, data_asc);
+    plist[i] = createDummyOperator(1, 1, 1, data_asc, 1);
   }
 
   SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL);
@@ -484,7 +984,7 @@ TEST(testCase, sorted_merge_Test) {
 
   while(1) {
     int64_t s = taosGetTimestampUs();
-    pRes = pOperator->exec(pOperator, &newgroup);
+    pRes = pOperator->nextDataFn(pOperator, &newgroup);
 
     int64_t e = taosGetTimestampUs();
     if (t++ == 1) {
@@ -507,15 +1007,13 @@ TEST(testCase, sorted_merge_Test) {
   int64_t s2 = taosGetTimestampUs();
   printf("total:%ld\n", s2 - s1);
 
-  pOperator->cleanupFn(pOperator->info, 2);
+  pOperator->closeFn(pOperator->info, 2);
   tfree(exp);
   tfree(exp1);
   taosArrayDestroy(pExprInfo);
   taosArrayDestroy(pOrderVal);
 }
 
-#endif
-
 TEST(testCase, time_interval_Operator_Test) {
   srand(time(NULL));
 
@@ -548,7 +1046,11 @@ TEST(testCase, time_interval_Operator_Test) {
   SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2);
 
   SExecTaskInfo ti = {0};
-  SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti);
+  SInterval interval = {0};
+  interval.sliding = interval.interval = 1000;
+  interval.slidingUnit = interval.intervalUnit = 'a';
+
+  SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &interval, &ti);
 
   bool newgroup = false;
   SSDataBlock* pRes = NULL;
@@ -560,7 +1062,7 @@ TEST(testCase, time_interval_Operator_Test) {
 
   while(1) {
     int64_t s = taosGetTimestampUs();
-    pRes = pOperator->exec(pOperator, &newgroup);
+    pRes = pOperator->nextDataFn(pOperator, &newgroup);
 
     int64_t e = taosGetTimestampUs();
     if (t++ == 1) {
@@ -583,7 +1085,7 @@ TEST(testCase, time_interval_Operator_Test) {
   int64_t s2 = taosGetTimestampUs();
   printf("total:%ld\n", s2 - s1);
 
-  pOperator->cleanupFn(pOperator->info, 2);
+  pOperator->closeFn(pOperator->info, 2);
   tfree(exp);
   tfree(exp1);
   taosArrayDestroy(pExprInfo);

source/libs/function/src/tpercentile.c

@@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
 
   resetSlotInfo(pBucket);
 
-  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp");
+  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp");
   if (ret != 0) {
     tMemBucketDestroy(pBucket);
     return NULL;

source/libs/parser/src/insertParser.c

@@ -153,7 +153,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) {
     if (NULL == dst) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
-    taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
+    taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
     dst->numOfTables = src->numOfTables;
     dst->size = src->size;
     TSWAP(dst->pData, src->pData, char*);

source/util/src/tcache.c

@@ -15,11 +15,99 @@
 
 #define _DEFAULT_SOURCE
 #include "tcache.h"
+#include "taoserror.h"
 #include "tlog.h"
-#include "ttimer.h"
 #include "tutil.h"
 
-static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) {
+#define CACHE_MAX_CAPACITY 1024*1024*16
+#define CACHE_DEFAULT_CAPACITY 1024*4
+
+static pthread_t       cacheRefreshWorker = {0};
+static pthread_once_t  cacheThreadInit = PTHREAD_ONCE_INIT;
+static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER;
+static SArray         *pCacheArrayList = NULL;
+static bool            stopRefreshWorker = false;
+static bool            refreshWorkerNormalStopped = false;
+static bool            refreshWorkerUnexpectedStopped = false;
+
+typedef struct SCacheNode {
+  uint64_t           addedTime;   // the added time when this element is added or updated into cache
+  uint64_t           lifespan;    // life duration when this element should be remove from cache
+  int64_t            expireTime;  // expire time
+  uint64_t           signature;
+  struct STrashElem *pTNodeHeader;    // point to trash node head
+  uint16_t           keyLen: 15;    // max key size: 32kb
+  bool               inTrashcan : 1;  // denote if it is in trash or not
+  uint32_t           size;            // allocated size for current SCacheNode
+  uint32_t           dataLen;
+  T_REF_DECLARE()
+  struct SCacheNode *pNext;
+  char              *key;
+  char              *data;
+} SCacheNode;
+
+typedef struct SCacheEntry {
+  int32_t     num;      // number of elements in current entry
+  SRWLatch    latch;    // entry latch
+  SCacheNode *next;
+} SCacheEntry;
+
+typedef struct STrashElem {
+  struct STrashElem *prev;
+  struct STrashElem *next;
+  SCacheNode        *pData;
+} STrashElem;
+
+typedef struct SCacheIter {
+  SCacheObj   *pCacheObj;
+  SCacheNode **pCurrent;
+  int32_t      entryIndex;
+  int32_t      index;
+  int32_t      numOfObj;
+} SCacheIter;
+
+/*
+ * to accommodate the old data which has the same key value of new one in hashList
+ * when an new node is put into cache, if an existed one with the same key:
+ * 1. if the old one does not be referenced, update it.
+ * 2. otherwise, move the old one to pTrash, addedTime the new one.
+ *
+ * when the node in pTrash does not be referenced, it will be release at the expired expiredTime
+ */
+struct SCacheObj {
+  int64_t            sizeInBytes;  // total allocated buffer in this hash table, SCacheObj is not included.
+  int64_t            refreshTime;
+  char              *name;
+  SCacheStatis       statistics;
+
+  SCacheEntry       *pEntryList;
+  size_t             capacity;     // number of slots
+  size_t             numOfElems;         // number of elements in cache
+  _hash_fn_t         hashFp;       // hash function
+  __cache_free_fn_t  freeFp;
+
+  uint32_t           numOfElemsInTrash;  // number of element in trash
+  STrashElem        *pTrash;
+
+  uint8_t            deleting;           // set the deleting flag to stop refreshing ASAP.
+  pthread_t          refreshWorker;
+  bool               extendLifespan;  // auto extend life span when one item is accessed.
+  int64_t            checkTick;       // tick used to record the check times of the refresh threads
+#if defined(LINUX)
+  pthread_rwlock_t lock;
+#else
+  pthread_mutex_t lock;
+#endif
+};
+
+typedef struct SCacheObjTravSup {
+  SCacheObj        *pCacheObj;
+  int64_t           time;
+  __cache_trav_fn_t fp;
+  void             *param1;
+} SCacheObjTravSup;
+
+static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) {
 #if defined(LINUX)
   pthread_rwlock_wrlock(&pCacheObj->lock);
 #else
@@ -27,7 +115,7 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) {
 #endif
 }
 
-static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) {
+static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) {
 #if defined(LINUX)
   pthread_rwlock_unlock(&pCacheObj->lock);
 #else
@@ -35,7 +123,7 @@ static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) {
 #endif
 }
 
-static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) {
+static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) {
 #if defined(LINUX)
   return pthread_rwlock_init(&pCacheObj->lock, NULL);
 #else
@@ -43,7 +131,7 @@ static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) {
 #endif
 }
 
-static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) {
+static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) {
 #if defined(LINUX)
   pthread_rwlock_destroy(&pCacheObj->lock);
 #else
@@ -63,14 +151,6 @@ static void doCleanupDataCache(SCacheObj *pCacheObj);
  */
 static void *taosCacheTimedRefresh(void *handle);
 
-static pthread_t       cacheRefreshWorker = {0};
-static pthread_once_t  cacheThreadInit = PTHREAD_ONCE_INIT;
-static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER;
-static SArray         *pCacheArrayList = NULL;
-static bool            stopRefreshWorker = false;
-static bool            refreshWorkerNormalStopped = false;
-static bool            refreshWorkerUnexpectedStopped = false;
-
 static void doInitRefreshThread(void) {
   pCacheArrayList = taosArrayInit(4, POINTER_BYTES);
 
@@ -99,9 +179,9 @@ pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) {
  *                 in pData. Pointer copy causes memory access error.
  * @param size     size of block
  * @param lifespan total survial expiredTime from now
- * @return         SCacheDataNode
+ * @return         SCacheNode
  */
-static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
+static SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
                                            uint64_t duration);
 
 /**
@@ -110,7 +190,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
  * @param pCacheObj    Cache object
  * @param pNode   Cache slot object
  */
-static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode);
+static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode);
 
 /**
  * remove nodes in trash with refCount == 0 in cache
@@ -126,18 +206,16 @@ static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force);
  * @param pCacheObj      cache object
  * @param pNode          data node
  */
-static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
+static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *pNode) {
   if (pNode->signature != (uint64_t)pNode) {
     uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode);
     return;
   }
 
-  atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
-  int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
-  assert(size > 0);
+  atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
 
   uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, total num:%d size:%" PRId64 "bytes",
-         pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize);
+         pCacheObj->name, pNode->key, pNode->data, pNode->size, (int)pCacheObj->numOfElems - 1, pCacheObj->sizeInBytes);
 
   if (pCacheObj->freeFp) {
     pCacheObj->freeFp(pNode->data);
@@ -181,6 +259,97 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem
   free(pElem);
 }
 
+static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) {
+  assert(pNode != NULL && pEntry != NULL);
+
+  pNode->pNext = pEntry->next;
+  pEntry->next = pNode;
+  pEntry->num += 1;
+}
+
+static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pNode) {
+  if (prev == NULL) {
+    ASSERT(pe->next == pNode);
+    pe->next = pNode->pNext;
+  } else {
+    prev->pNext = pNode->pNext;
+  }
+
+  pNode->pNext = NULL;
+  pe->num -= 1;
+}
+
+static FORCE_INLINE SCacheEntry* doFindEntry(SCacheObj* pCacheObj, const void* key, size_t keyLen) {
+  uint32_t hashVal = (*pCacheObj->hashFp)(key, keyLen);
+  int32_t  slot = hashVal % pCacheObj->capacity;
+  return &pCacheObj->pEntryList[slot];
+}
+
+static FORCE_INLINE SCacheNode *
+doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode** prev) {
+  SCacheNode *pNode = pe->next;
+  while (pNode) {
+    if ((pNode->keyLen == keyLen) && memcmp(pNode->key, key, keyLen) == 0) {
+      break;
+    }
+    *prev = pNode;
+    pNode = pNode->pNext;
+  }
+
+  return pNode;
+}
+
+static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) {
+  SCacheObjTravSup *ps = (SCacheObjTravSup *)param;
+  SCacheObj     *pCacheObj = ps->pCacheObj;
+
+  if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) {
+    taosCacheReleaseNode(pCacheObj, pNode);
+
+    // this node should be remove from hash table
+    return false;
+  }
+
+  if (ps->fp) {
+    (ps->fp)(pNode->data, ps->param1);
+  }
+
+  // do not remove element in hash table
+  return true;
+}
+
+static bool doRemoveNodeFn(void *param, SCacheNode *pNode) {
+  SCacheObjTravSup *ps = (SCacheObjTravSup *)param;
+  SCacheObj     *pCacheObj = ps->pCacheObj;
+
+  if (T_REF_VAL_GET(pNode) == 0) {
+    taosCacheReleaseNode(pCacheObj, pNode);
+  } else {  // do add to trashcan
+    taosAddToTrashcan(pCacheObj, pNode);
+  }
+
+  // this node should be remove from hash table
+  return false;
+}
+
+static FORCE_INLINE int32_t getCacheCapacity(int32_t length) {
+  int32_t len = 0;
+  if (length < CACHE_DEFAULT_CAPACITY) {
+    len = CACHE_DEFAULT_CAPACITY;
+    return len;
+  } else if (length > CACHE_MAX_CAPACITY) {
+    len = CACHE_MAX_CAPACITY;
+    return len;
+  }
+
+  len = CACHE_DEFAULT_CAPACITY;
+  while (len < length && len < CACHE_MAX_CAPACITY) {
+    len = (len << 1u);
+  }
+
+  return len > CACHE_MAX_CAPACITY? CACHE_MAX_CAPACITY:len;
+}
+
 SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName) {
   const int32_t SLEEP_DURATION = 500;  // 500 ms
@@ -195,39 +364,42 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
     return NULL;
   }
 
-  pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK);
-  pCacheObj->name = strdup(cacheName);
-  if (pCacheObj->pHashTable == NULL) {
+  // TODO add the auto extend procedure
+  pCacheObj->capacity = 4096;
+  pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry));
+  if (pCacheObj->pEntryList == NULL) {
     free(pCacheObj);
     uError("failed to allocate memory, reason:%s", strerror(errno));
     return NULL;
   }
 
   // set free cache node callback function
-  pCacheObj->freeFp = fn;
-  pCacheObj->refreshTime = refreshTimeInSeconds * 1000;
-  pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION;
+  pCacheObj->hashFp         = taosGetDefaultHashFunction(keyType);
+  pCacheObj->freeFp         = fn;
+  pCacheObj->refreshTime    = refreshTimeInSeconds * 1000;
+  pCacheObj->checkTick      = pCacheObj->refreshTime / SLEEP_DURATION;
   pCacheObj->extendLifespan = extendLifespan;  // the TTL after the last access
 
-  if (__cache_lock_init(pCacheObj) != 0) {
-    taosHashCleanup(pCacheObj->pHashTable);
+  if (__trashcan_lock_init(pCacheObj) != 0) {
+    tfree(pCacheObj->pEntryList);
     free(pCacheObj);
 
     uError("failed to init lock, reason:%s", strerror(errno));
     return NULL;
   }
 
+  pCacheObj->name = strdup(cacheName);
   doRegisterCacheObj(pCacheObj);
   return pCacheObj;
 }
 
 void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize,
                    int32_t durationMS) {
-  if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) {
+  if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) {
     return NULL;
   }
 
-  SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS);
+  SCacheNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS);
   if (pNode1 == NULL) {
     uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
     return NULL;
@@ -235,86 +407,77 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
 
   T_REF_INC(pNode1);
 
-  int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
-  if (succ == 0) {
-    atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size);
-    uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
-           ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes",
-           pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime,
-           (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
-  } else {  // duplicated key exists
-    while (1) {
-      SCacheDataNode *p = NULL;
-      //      int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*));
-      int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen);
-
-      // add to trashcan
-      if (ret == 0) {
-        if (T_REF_VAL_GET(p) == 0) {
-          if (pCacheObj->freeFp) {
-            pCacheObj->freeFp(p->data);
-          }
+  SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen);
 
-          atomic_sub_fetch_64(&pCacheObj->totalSize, p->size);
-          tfree(p);
-        } else {
-          taosAddToTrashcan(pCacheObj, p);
-          uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
-        }
-      }
-
-      assert(T_REF_VAL_GET(pNode1) == 1);
-
-      ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
-      if (ret == 0) {
-        atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size);
+  taosWLockLatch(&pe->latch);
 
-        uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
-               ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes",
-               pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime,
-               (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
+  SCacheNode *prev  = NULL;
+  SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev);
 
-        return pNode1->data;
+  if (pNode == NULL) {
+    pushfrontNodeInEntryList(pe, pNode1);
+    atomic_add_fetch_64(&pCacheObj->numOfElems, 1);
+    atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
+    uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
+           ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
+           pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
+           pCacheObj->sizeInBytes, (int64_t)dataSize);
+  } else {  // duplicated key exists
+    // move current node to trashcan
+    removeNodeInEntryList(pe, prev, pNode);
 
-      } else {
-        // failed, try again
+    if (T_REF_VAL_GET(pNode) == 0) {
+      if (pCacheObj->freeFp) {
+        pCacheObj->freeFp(pNode->data);
       }
+
+      atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
+      tfree(pNode);
+    } else {
+      taosAddToTrashcan(pCacheObj, pNode);
+      uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data);
     }
+
+    pushfrontNodeInEntryList(pe, pNode1);
+    atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
+    uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
+           ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
+           pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
+           pCacheObj->sizeInBytes, (int64_t)dataSize);
   }
 
+  taosWUnLockLatch(&pe->latch);
   return pNode1->data;
 }
 
-static void incRefFn(void *ptNode) {
-  assert(ptNode != NULL);
-
-  SCacheDataNode **p = (SCacheDataNode **)ptNode;
-  assert(T_REF_VAL_GET(*p) >= 0);
-
-  int32_t ret = T_REF_INC(*p);
-  assert(ret > 0);
-}
-
 void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) {
   if (pCacheObj == NULL || pCacheObj->deleting == 1) {
     return NULL;
   }
 
-  if (taosHashGetSize(pCacheObj->pHashTable) == 0) {
+  if (pCacheObj->numOfElems == 0) {
     atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
     return NULL;
   }
 
-  SCacheDataNode *ptNode = NULL;
-  taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode);
-  //  taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode);
+  SCacheNode *prev = NULL;
+  SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen);
+
+  taosRLockLatch(&pe->latch);
+
+  SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev);
+  if (pNode != NULL) {
+    int32_t ref = T_REF_INC(pNode);
+    ASSERT(ref > 0);
+  }
 
-  void *pData = (ptNode != NULL) ? ptNode->data : NULL;
+  taosRUnLockLatch(&pe->latch);
 
+  void *pData = (pNode != NULL) ? pNode->data : NULL;
   if (pData != NULL) {
     atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1);
     uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData,
-           T_REF_VAL_GET(ptNode));
+           T_REF_VAL_GET(pNode));
   } else {
     atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
     uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key);
@@ -327,9 +490,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen
 void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
   if (pCacheObj == NULL || data == NULL) return NULL;
 
-  size_t          offset = offsetof(SCacheDataNode, data);
-  SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset);
-
+  SCacheNode *ptNode = (SCacheNode *)((char *)data - sizeof(SCacheNode));
   if (ptNode->signature != (uint64_t)ptNode) {
     uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
     return NULL;
@@ -343,24 +504,20 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
   return data;
 }
 
-void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
+void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) {
   if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
 
-  size_t          offset = offsetof(SCacheDataNode, data);
-  SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset);
-
+  SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
   if (ptNode->signature != (uint64_t)ptNode) {
     uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
     return NULL;
   }
 
   assert(T_REF_VAL_GET(ptNode) >= 1);
-
   char *d = *data;
 
   // clear its reference to old area
   *data = NULL;
-
   return d;
 }
 
@@ -378,9 +535,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   // therefore the check for the empty of both the hash table and the trashcan has a race condition.
   // It happens when there is only one object in the cache, and two threads which has referenced this object
   // start to free the it simultaneously [TD-1569].
-  size_t offset = offsetof(SCacheDataNode, data);
-
-  SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset);
+  SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
   if (pNode->signature != (uint64_t)pNode) {
     uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode);
     return;
@@ -419,9 +574,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
         // destroyed by refresh worker if decrease ref count before removing it from linked-list.
         assert(pNode->pTNodeHeader->pData == pNode);
 
-        __cache_wr_lock(pCacheObj);
+        __trashcan_wr_lock(pCacheObj);
         doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
-        __cache_unlock(pCacheObj);
+        __trashcan_unlock(pCacheObj);
 
         ref = T_REF_DEC(pNode);
         assert(ref == 0);
@@ -434,36 +589,37 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
     } else {
       // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread
       // when reaches here.
-      SCacheDataNode *p = NULL;
-      int32_t         ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize);
-      //      int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void
-      //      *));
+      SCacheNode * prev = NULL;
+      SCacheEntry *pe = doFindEntry(pCacheObj, pNode->key, pNode->keyLen);
+
+      taosWLockLatch(&pe->latch);
       ref = T_REF_DEC(pNode);
 
-      // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing.
-      // note that the remove operation can be executed only once.
-      if (ret == 0) {
+      SCacheNode *p = doSearchInEntryList(pe, pNode->key, pNode->keyLen, &prev);
+
+      if (p != NULL) {
+        // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing.
+        // note that the remove operation can be executed only once.
         if (p != pNode) {
           uDebug(
-              "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by "
-              "others already",
-              pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data);
+              "cache:%s, key:%p, a new entry:%p found, refcnt:%d, prev entry:%p, refcnt:%d has been removed by "
+              "others already, prev must in trashcan",
+              pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode));
 
-          assert(p->pTNodeHeader == NULL);
-          taosAddToTrashcan(pCacheObj, p);
+          assert(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL);
         } else {
+          removeNodeInEntryList(pe, prev, p);
           uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key,
                  pNode->data, ref);
           if (ref > 0) {
             assert(pNode->pTNodeHeader == NULL);
-
             taosAddToTrashcan(pCacheObj, pNode);
           } else {  // ref == 0
-            atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
+            atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
 
-            int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
+            int32_t size = (int32_t)pCacheObj->numOfElems;
             uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes",
-                   pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize);
+                   pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->sizeInBytes);
 
             if (pCacheObj->freeFp) {
               pCacheObj->freeFp(pNode->data);
@@ -472,6 +628,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
             free(pNode);
           }
         }
+
+        taosWUnLockLatch(&pe->latch);
       } else {
         uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name,
                pNode->key, pNode->data, ref);
@@ -483,59 +641,43 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
     char *key = pNode->key;
     char *p = pNode->data;
 
-    //    int32_t ref = T_REF_VAL_GET(pNode);
-    //
-    //    if (ref == 1 && inTrashcan) {
-    //      // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may
-    //      be
-    //      // destroyed by refresh worker if decrease ref count before removing it from linked-list.
-    //      assert(pNode->pTNodeHeader->pData == pNode);
-    //
-    //      __cache_wr_lock(pCacheObj);
-    //      doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
-    //      __cache_unlock(pCacheObj);
-    //
-    //      ref = T_REF_DEC(pNode);
-    //      assert(ref == 0);
-    //
-    //      doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader);
-    //    } else {
-    //      ref = T_REF_DEC(pNode);
-    //      assert(ref >= 0);
-    //    }
-
     int32_t ref = T_REF_DEC(pNode);
     uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan);
   }
 }
 
-typedef struct SHashTravSupp {
-  SCacheObj        *pCacheObj;
-  int64_t           time;
-  __cache_trav_fn_t fp;
-  void             *param1;
-} SHashTravSupp;
+void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) {
+  int32_t numOfEntries = (int32_t)pCacheObj->capacity;
+  for (int32_t i = 0; i < numOfEntries; ++i) {
+    SCacheEntry *pEntry = &pCacheObj->pEntryList[i];
+    if (pEntry->num == 0) {
+      continue;
+    }
 
-static bool travHashTableEmptyFn(void *param, void *data) {
-  SHashTravSupp *ps = (SHashTravSupp *)param;
-  SCacheObj     *pCacheObj = ps->pCacheObj;
+    taosWLockLatch(&pEntry->latch);
 
-  SCacheDataNode *pNode = *(SCacheDataNode **)data;
+    SCacheNode *pNode = pEntry->next;
+    while (pNode != NULL) {
+      SCacheNode *next = pNode->pNext;
 
-  if (T_REF_VAL_GET(pNode) == 0) {
-    taosCacheReleaseNode(pCacheObj, pNode);
-  } else {  // do add to trashcan
-    taosAddToTrashcan(pCacheObj, pNode);
-  }
+      if (fp(pSup, pNode)) {
+        pNode = pNode->pNext;
+      } else {
+        pEntry->next = next;
+        pEntry->num -= 1;
 
-  // this node should be remove from hash table
-  return false;
-}
+        atomic_sub_fetch_64(&pCacheObj->numOfElems, 1);
+        pNode = next;
+      }
+    }
 
-void taosCacheEmpty(SCacheObj *pCacheObj) {
-  SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
+    taosWUnLockLatch(&pEntry->latch);
+  }
+}
 
-  //  taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
+void taosCacheEmpty(SCacheObj* pCacheObj) {
+  SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
+  doTraverseElems(pCacheObj, doRemoveNodeFn, &sup);
   taosTrashcanEmpty(pCacheObj, false);
 }
 
@@ -558,38 +700,41 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
   doCleanupDataCache(pCacheObj);
 }
 
-SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) {
-  size_t totalSize = size + sizeof(SCacheDataNode) + keyLen;
+SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) {
+  size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen;
 
-  SCacheDataNode *pNewNode = calloc(1, totalSize);
+  SCacheNode *pNewNode = calloc(1, sizeInBytes);
   if (pNewNode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     uError("failed to allocate memory, reason:%s", strerror(errno));
     return NULL;
   }
 
+  pNewNode->data    = (char*)pNewNode + sizeof(SCacheNode);
+  pNewNode->dataLen = size;
   memcpy(pNewNode->data, pData, size);
 
-  pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
-  pNewNode->keySize = (uint16_t)keyLen;
+  pNewNode->key    = (char *)pNewNode + sizeof(SCacheNode) + size;
+  pNewNode->keyLen = (uint16_t)keyLen;
 
   memcpy(pNewNode->key, key, keyLen);
 
-  pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
-  pNewNode->lifespan = duration;
+  pNewNode->addedTime  = (uint64_t)taosGetTimestampMs();
+  pNewNode->lifespan   = duration;
   pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan;
-  pNewNode->signature = (uint64_t)pNewNode;
-  pNewNode->size = (uint32_t)totalSize;
+  pNewNode->signature  = (uint64_t)pNewNode;
+  pNewNode->size       = (uint32_t)sizeInBytes;
 
   return pNewNode;
 }
 
-void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
+void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) {
   if (pNode->inTrashcan) { /* node is already in trash */
     assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode);
     return;
   }
 
-  __cache_wr_lock(pCacheObj);
+  __trashcan_wr_lock(pCacheObj);
   STrashElem *pElem = calloc(1, sizeof(STrashElem));
   pElem->pData = pNode;
   pElem->prev = NULL;
@@ -604,14 +749,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
 
   pCacheObj->pTrash = pElem;
   pCacheObj->numOfElemsInTrash++;
-  __cache_unlock(pCacheObj);
+  __trashcan_unlock(pCacheObj);
 
   uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key,
          pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }
 
 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
-  __cache_wr_lock(pCacheObj);
+  __trashcan_wr_lock(pCacheObj);
 
   if (pCacheObj->numOfElemsInTrash == 0) {
     if (pCacheObj->pTrash != NULL) {
@@ -620,7 +765,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
              pCacheObj->numOfElemsInTrash);
     }
 
-    __cache_unlock(pCacheObj);
+    __trashcan_unlock(pCacheObj);
     return;
   }
 
@@ -645,49 +790,28 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
     }
   }
 
-  __cache_unlock(pCacheObj);
+  __trashcan_unlock(pCacheObj);
 }
 
 void doCleanupDataCache(SCacheObj *pCacheObj) {
-  //  SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
-  //  taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
+  SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
+  doTraverseElems(pCacheObj, doRemoveNodeFn, &sup);
 
   // todo memory leak if there are object with refcount greater than 0 in hash table?
-  taosHashCleanup(pCacheObj->pHashTable);
   taosTrashcanEmpty(pCacheObj, true);
 
-  __cache_lock_destroy(pCacheObj);
+  __trashcan_lock_destroy(pCacheObj);
 
+  tfree(pCacheObj->pEntryList);
   tfree(pCacheObj->name);
-  memset(pCacheObj, 0, sizeof(SCacheObj));
   free(pCacheObj);
 }
 
-bool travHashTableFn(void *param, void *data) {
-  SHashTravSupp *ps = (SHashTravSupp *)param;
-  SCacheObj     *pCacheObj = ps->pCacheObj;
-
-  SCacheDataNode *pNode = *(SCacheDataNode **)data;
-  if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) {
-    taosCacheReleaseNode(pCacheObj, pNode);
-
-    // this node should be remove from hash table
-    return false;
-  }
-
-  if (ps->fp) {
-    (ps->fp)(pNode->data, ps->param1);
-  }
-
-  // do not remove element in hash table
-  return true;
-}
-
 static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) {
   assert(pCacheObj != NULL);
 
-  SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1};
-  //  taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup);
+  SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1};
+  doTraverseElems(pCacheObj, doRemoveExpiredFn, &sup);
 }
 
 void taosCacheRefreshWorkerUnexpectedStopped(void) {
@@ -746,7 +870,7 @@ void *taosCacheTimedRefresh(void *handle) {
         continue;
       }
 
-      size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable);
+      size_t elemInHash = pCacheObj->numOfElems;
       if (elemInHash + pCacheObj->numOfElemsInTrash == 0) {
         continue;
       }
@@ -784,4 +908,94 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1)
   doCacheRefresh(pCacheObj, now, fp, param1);
 }
 
-void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; }
\ No newline at end of file
+void taosStopCacheRefreshWorker(void) {
+  stopRefreshWorker = true;
+}
+
+size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) {
+  return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash;
+}
+
+SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) {
+  ASSERT(pCacheObj != NULL);
+  SCacheIter* pIter = calloc(1, sizeof(SCacheIter));
+  pIter->pCacheObj  = (SCacheObj*) pCacheObj;
+  pIter->entryIndex = -1;
+  pIter->index      = -1;
+  return pIter;
+}
+
+bool taosCacheIterNext(SCacheIter* pIter) {
+  SCacheObj* pCacheObj = pIter->pCacheObj;
+
+  if (pIter->index + 1 >= pIter->numOfObj) {
+    if (pIter->entryIndex + 1 >= pCacheObj->capacity) {
+      return false;
+    }
+
+    // release the reference for all objects in the snapshot
+    for(int32_t i = 0; i < pIter->numOfObj; ++i) {
+      char* p= pIter->pCurrent[i]->data;
+      taosCacheRelease(pCacheObj, (void**) &p, false);
+      pIter->pCurrent[i] = NULL;
+    }
+
+    while(1) {
+      SCacheEntry *pEntry = &pCacheObj->pEntryList[++pIter->entryIndex];
+      taosRLockLatch(&pEntry->latch);
+
+      if (pEntry->num == 0) {
+        taosRUnLockLatch(&pEntry->latch);
+        continue;
+      }
+
+      if (pIter->numOfObj < pEntry->num) {
+        char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES);
+        if (tmp == NULL) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          taosRUnLockLatch(&pEntry->latch);
+          return false;
+        }
+
+        pIter->pCurrent = (SCacheNode **)tmp;
+      }
+
+      SCacheNode* pNode = pEntry->next;
+      for (int32_t i = 0; i < pEntry->num; ++i) {
+        ASSERT(pNode != NULL);
+
+        pIter->pCurrent[i] = pNode;
+        int32_t ref = T_REF_INC(pIter->pCurrent[i]);
+        ASSERT(ref >= 1);
+
+        pNode = pNode->pNext;
+      }
+
+      pIter->numOfObj = pEntry->num;
+      taosRUnLockLatch(&pEntry->latch);
+
+      pIter->index    = -1;
+      break;
+    }
+  }
+
+  pIter->index += 1;
+  return true;
+}
+
+void* taosCacheIterGetData(const SCacheIter* pIter, size_t* len) {
+  SCacheNode* pNode = pIter->pCurrent[pIter->index];
+  *len = pNode->dataLen;
+  return pNode->data;
+}
+
+void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) {
+  SCacheNode* pNode = pIter->pCurrent[pIter->index];
+  *len = pNode->keyLen;
+  return pNode->key;
+}
+
+void taosCacheDestroyIter(SCacheIter* pIter) {
+  tfree(pIter->pCurrent);
+  tfree(pIter);
+}
\ No newline at end of file

source/util/src/thash.c

@@ -15,69 +15,126 @@
 
 #define _DEFAULT_SOURCE
 #include "thash.h"
-#include "tdef.h"
+#include "taoserror.h"
+#include "os.h"
 #include "tlog.h"
 
 // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT
-#define MAX_WARNING_REF_COUNT 10000
-#define EXT_SIZE              1024
-#define HASH_NEED_RESIZE(_h)  ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
-
-#define DO_FREE_HASH_NODE(_n) \
-  do {                        \
-    tfree(_n);                \
-  } while (0)
-
-#define FREE_HASH_NODE(_h, _n)              \
-  do {                                      \
-    if ((_h)->freeFp) {                     \
-      (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \
-    }                                       \
-                                            \
-    DO_FREE_HASH_NODE(_n);                  \
+#define MAX_WARNING_REF_COUNT    10000
+#define HASH_MAX_CAPACITY        (1024 * 1024 * 16)
+#define HASH_DEFAULT_LOAD_FACTOR (0.75)
+#define HASH_INDEX(v, c)         ((v) & ((c)-1))
+
+#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
+
+#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
+#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
+#define GET_HASH_PNODE(_n)     ((SHashNode *)((char*)(_n) - sizeof(SHashNode)))
+
+#define FREE_HASH_NODE(_n) \
+  do {                     \
+    tfree(_n);             \
   } while (0);
 
-static FORCE_INLINE void __wr_lock(void *lock, int32_t type) {
-  if (type == HASH_NO_LOCK) {
+typedef struct SHashEntry {
+  int32_t    num;      // number of elements in current entry
+  SRWLatch   latch;    // entry latch
+  SHashNode *next;
+} SHashEntry;
+
+typedef struct SHashObj {
+  SHashEntry     **hashList;
+  size_t           capacity;     // number of slots
+  size_t           size;         // number of elements in hash table
+  _hash_fn_t       hashFp;       // hash function
+  _equal_fn_t      equalFp;      // equal function
+  _hash_free_fn_t  freeFp;       // hash node free callback function
+  SRWLatch         lock;         // read-write spin lock
+  SHashLockTypeE   type;         // lock type
+  bool             enableUpdate; // enable update
+  SArray          *pMemBlock;    // memory block allocated for SHashEntry
+  _hash_before_fn_t callbackFp;  // function invoked before return the value to caller
+} SHashObj;
+
+/*
+ * Function definition
+ */
+static FORCE_INLINE void taosHashWLock(SHashObj *pHashObj) {
+  if (pHashObj->type == HASH_NO_LOCK) {
+    return;
+  }
+  taosWLockLatch(&pHashObj->lock);
+}
+
+static FORCE_INLINE void taosHashWUnlock(SHashObj *pHashObj) {
+  if (pHashObj->type == HASH_NO_LOCK) {
+    return;
+  }
+
+  taosWUnLockLatch(&pHashObj->lock);
+}
+
+static FORCE_INLINE void taosHashRLock(SHashObj *pHashObj) {
+  if (pHashObj->type == HASH_NO_LOCK) {
+    return;
+  }
+
+  taosRLockLatch(&pHashObj->lock);
+}
+
+static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) {
+  if (pHashObj->type == HASH_NO_LOCK) {
+    return;
+  }
+
+  taosRUnLockLatch(&pHashObj->lock);
+}
+
+static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) {
+  if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
-  taosWLockLatch(lock);
+  taosWLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void __rd_lock(void *lock, int32_t type) {
-  if (type == HASH_NO_LOCK) {
+static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
+  if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
-  taosRLockLatch(lock);
+
+  taosWUnLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) {
-  if (type == HASH_NO_LOCK) {
+static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) {
+  if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
-  taosRUnLockLatch(lock);
+
+  taosRLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) {
-  if (type == HASH_NO_LOCK) {
+static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
+  if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
-  taosWUnLockLatch(lock);
+
+  taosRUnLockLatch(&pe->latch);
 }
 
 static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
-  int32_t len = TMIN(length, HASH_MAX_CAPACITY);
+  int32_t len = MIN(length, HASH_MAX_CAPACITY);
 
   int32_t i = 4;
   while (i < len) i = (i << 1u);
   return i;
 }
 
-static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen,
-                                                   uint32_t hashVal) {
+static FORCE_INLINE SHashNode *
+doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
+    if ((pNode->keyLen == keyLen) &&
+        ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
         pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
@@ -90,60 +147,57 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr
 }
 
 /**
- * Resize the hash list if the threshold is reached
+ * resize the hash list if the threshold is reached
  *
  * @param pHashObj
  */
 static void taosHashTableResize(SHashObj *pHashObj);
 
 /**
+ * allocate and initialize a hash node
+ *
  * @param key      key of object for hash, usually a null-terminated string
  * @param keyLen   length of key
- * @param pData    actually data. Requires a consecutive memory block, no pointer is allowed in pData.
- *                 Pointer copy causes memory access error.
+ * @param pData    data to be stored in hash node
  * @param dsize    size of data
  * @return         SHashNode
  */
 static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal);
 
 /**
- * Update the hash node
+ * update the hash node
  *
- * @param pNode   hash node
- * @param key     key for generate hash value
- * @param keyLen  key length
- * @param pData   actual data
- * @param dsize   size of actual data
- * @return        hash node
+ * @param pHashObj   hash table object
+ * @param pe         hash table entry to operate on
+ * @param prev       previous node
+ * @param pNode      the old node with requested key
+ * @param pNewNode   the new node with requested key
  */
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode,
-                                                SHashNode *pNewNode) {
+static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
   assert(pNode->keyLen == pNewNode->keyLen);
 
-  pNode->count--;
+  atomic_sub_fetch_32(&pNode->refCount, 1);
   if (prev != NULL) {
     prev->next = pNewNode;
   } else {
     pe->next = pNewNode;
   }
 
-  if (pNode->count <= 0) {
+  if (pNode->refCount <= 0) {
     pNewNode->next = pNode->next;
-    DO_FREE_HASH_NODE(pNode);
+    FREE_HASH_NODE(pNode);
   } else {
     pNewNode->next = pNode;
     pe->num++;
-    atomic_add_fetch_32(&pHashObj->size, 1);
+    atomic_add_fetch_64(&pHashObj->size, 1);
   }
-
-  return pNewNode;
 }
 
 /**
  * insert the hash node at the front of the linked list
  *
- * @param pHashObj
- * @param pNode
+ * @param pHashObj   hash table object
+ * @param pNode      the old node with requested key
  */
 static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
 
@@ -156,47 +210,69 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
 static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj);
 
 /**
- * Get the next element in hash table for iterator
- * @param pIter
+ *
+ * @param pHashObj
  * @return
  */
+static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) {
+  return taosHashGetSize(pHashObj) == 0;
+}
 
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
-  assert(fn != NULL);
+  if (fn == NULL) {
+    assert(0);
+    return NULL;
+  }
+
   if (capacity == 0) {
     capacity = 4;
   }
 
   SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj));
   if (pHashObj == NULL) {
-    uError("failed to allocate memory, reason:%s", strerror(errno));
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
   // the max slots is not defined by user
   pHashObj->capacity = taosHashCapacity((int32_t)capacity);
-  assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
+
   pHashObj->equalFp = memcmp;
-  pHashObj->hashFp = fn;
+  pHashObj->hashFp  = fn;
   pHashObj->type = type;
   pHashObj->enableUpdate = update;
 
+  ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
+
   pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *));
   if (pHashObj->hashList == NULL) {
     free(pHashObj);
-    uError("failed to allocate memory, reason:%s", strerror(errno));
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
-  } else {
-    pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *));
+  }
 
-    void *p = calloc(pHashObj->capacity, sizeof(SHashEntry));
-    for (int32_t i = 0; i < pHashObj->capacity; ++i) {
-      pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry));
-    }
+  pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *));
+  if (pHashObj->pMemBlock == NULL) {
+    free(pHashObj->hashList);
+    free(pHashObj);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  void *p = calloc(pHashObj->capacity, sizeof(SHashEntry));
+  if (p == NULL) {
+    taosArrayDestroy(pHashObj->pMemBlock);
+    free(pHashObj->hashList);
+    free(pHashObj);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
 
-    taosArrayPush(pHashObj->pMemBlock, &p);
+  for (int32_t i = 0; i < pHashObj->capacity; ++i) {
+    pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry));
   }
 
+  taosArrayPush(pHashObj->pMemBlock, &p);
   return pHashObj;
 }
 
@@ -206,16 +282,24 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) {
   }
 }
 
+void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp) {
+  if (pHashObj != NULL && fp != NULL) {
+    pHashObj->freeFp = fp;
+  }
+}
+
 int32_t taosHashGetSize(const SHashObj *pHashObj) {
-  if (!pHashObj) {
+  if (pHashObj == NULL) {
     return 0;
   }
-  return (int32_t)atomic_load_32(&pHashObj->size);
+  return (int32_t)atomic_load_64(&pHashObj->size);
 }
 
-static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; }
+int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
+  if (pHashObj == NULL || key == NULL || keyLen == 0) {
+    return -1;
+  }
 
-int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) {
   uint32_t   hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
   SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal);
   if (pNewNode == NULL) {
@@ -224,30 +308,32 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void
 
   // need the resize process, write lock applied
   if (HASH_NEED_RESIZE(pHashObj)) {
-    __wr_lock((void *)&pHashObj->lock, pHashObj->type);
+    taosHashWLock(pHashObj);
     taosHashTableResize(pHashObj);
-    __wr_unlock((void *)&pHashObj->lock, pHashObj->type);
+    taosHashWUnlock(pHashObj);
   }
 
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
+  // disable resize
+  taosHashRLock(pHashObj);
 
   int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[slot];
 
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosWLockLatch(&pe->latch);
-  }
+  taosHashEntryWLock(pHashObj, pe);
 
   SHashNode *pNode = pe->next;
+#if 0
   if (pe->num > 0) {
     assert(pNode != NULL);
   } else {
     assert(pNode == NULL);
   }
+#endif
 
-  SHashNode *prev = NULL;
+  SHashNode* prev = NULL;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
+    if ((pNode->keyLen == keyLen) &&
+        (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
         pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
@@ -260,24 +346,13 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void
   if (pNode == NULL) {
     // no data in hash table with the specified key, add it into hash table
     pushfrontNodeInEntryList(pe, pNewNode);
+    assert(pe->next != NULL);
 
-    if (pe->num == 0) {
-      assert(pe->next == NULL);
-    } else {
-      assert(pe->next != NULL);
-    }
-
-    if (pHashObj->type == HASH_ENTRY_LOCK) {
-      taosWUnLockLatch(&pe->latch);
-    }
+    taosHashEntryWUnlock(pHashObj, pe);
 
     // enable resize
-    __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
-    atomic_add_fetch_32(&pHashObj->size, 1);
-
-    if (newAdded) {
-      *newAdded = true;
-    }
+    taosHashRUnlock(pHashObj);
+    atomic_add_fetch_64(&pHashObj->size, 1);
 
     return 0;
   } else {
@@ -285,131 +360,66 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void
     if (pHashObj->enableUpdate) {
       doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode);
     } else {
-      DO_FREE_HASH_NODE(pNewNode);
+      FREE_HASH_NODE(pNewNode);
     }
 
-    if (pHashObj->type == HASH_ENTRY_LOCK) {
-      taosWUnLockLatch(&pe->latch);
-    }
+    taosHashEntryWUnlock(pHashObj, pe);
 
     // enable resize
-    __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
-
-    if (newAdded) {
-      *newAdded = false;
-    }
-
-    return pHashObj->enableUpdate ? 0 : -2;
+    taosHashRUnlock(pHashObj);
+    return pHashObj->enableUpdate ? 0 : -1;
   }
 }
 
-int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
-  return taosHashPutImpl(pHashObj, key, keyLen, data, size, NULL);
-}
-
-int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) {
-  return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded);
-}
+static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef);
 
 void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
-  return taosHashGetClone(pHashObj, key, keyLen, NULL);
+  void* p = NULL;
+  return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false);
 }
 
-// TODO(yihaoDeng), merge with taosHashGetClone
-void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d,
-                          size_t *sz) {
-  if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
-    return NULL;
-  }
-
-  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
-
-  // only add the read lock to disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
-
-  int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
-  SHashEntry *pe = pHashObj->hashList[slot];
-
-  // no data, return directly
-  if (atomic_load_32(&pe->num) == 0) {
-    __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
-    return NULL;
-  }
-
-  char *data = NULL;
-
-  // lock entry
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosRLockLatch(&pe->latch);
-  }
-
-  if (pe->num > 0) {
-    assert(pe->next != NULL);
-  } else {
-    assert(pe->next == NULL);
-  }
-
-  SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal);
-  if (pNode != NULL) {
-    if (fp != NULL) {
-      fp(GET_HASH_NODE_DATA(pNode));
-    }
-
-    if (*d == NULL) {
-      *sz = pNode->dataLen + EXT_SIZE;
-      *d = calloc(1, *sz);
-    } else if (*sz < pNode->dataLen) {
-      *sz = pNode->dataLen + EXT_SIZE;
-      *d = realloc(*d, *sz);
-    }
-    memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen);
-    // just make runtime happy
-    if ((*sz) - pNode->dataLen > 0) {
-      memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen);
-    }
-
-    data = GET_HASH_NODE_DATA(pNode);
-  }
+int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) {
+  terrno = 0;
+  /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
+  return terrno;
+}
 
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosRUnLockLatch(&pe->latch);
-  }
+int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) {
+  terrno = 0;
 
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
-  return data;
+  /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
+  return terrno;
 }
 
-void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *d, bool acquire) {
-  if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
+void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) {
+  if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
     return NULL;
   }
 
   uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
 
   // only add the read lock to disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashRLock(pHashObj);
 
   int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[slot];
 
   // no data, return directly
   if (atomic_load_32(&pe->num) == 0) {
-    __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
+    taosHashRUnlock(pHashObj);
     return NULL;
   }
 
   char *data = NULL;
+  taosHashEntryRLock(pHashObj, pe);
 
-  // lock entry
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosRLockLatch(&pe->latch);
-  }
-
+#if 0
   if (pe->num > 0) {
     assert(pe->next != NULL);
   } else {
     assert(pe->next == NULL);
   }
+#endif
 
   SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal);
   if (pNode != NULL) {
@@ -417,173 +427,106 @@ void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v
       pHashObj->callbackFp(GET_HASH_NODE_DATA(pNode));
     }
 
-    if (d != NULL) {
-      memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen);
+    if (size != NULL) {
+      if (*d == NULL) {
+        *size =  pNode->dataLen;
+        *d = calloc(1, *size);
+        if (*d == NULL) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return NULL;
+        }
+      } else if (*size < pNode->dataLen) {
+        *size =  pNode->dataLen;
+        char* tmp = realloc(*d, *size);
+        if (tmp == NULL) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          return NULL;
+        }
+
+        *d = tmp;
+      }
+    }
+
+    if (addRef) {
+      atomic_add_fetch_16(&pNode->refCount, 1);
     }
 
-    if (acquire) {
-      atomic_add_fetch_16(&pNode->count, 1);
+    if (*d != NULL) {
+      memcpy(*d, GET_HASH_NODE_DATA(pNode), pNode->dataLen);
     }
 
     data = GET_HASH_NODE_DATA(pNode);
   }
 
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosRUnLockLatch(&pe->latch);
-  }
+  taosHashEntryRUnlock(pHashObj, pe);
+  taosHashRUnlock(pHashObj);
 
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
   return data;
 }
 
-void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *d) {
-  return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false);
-}
-
-void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
-  return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true);
-}
-
-int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen /*, void *data, size_t dsize*/) {
-  if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
+int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) {
+  if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) {
     return -1;
   }
 
   uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
 
   // disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashRLock(pHashObj);
 
   int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[slot];
 
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosWLockLatch(&pe->latch);
-  }
+  taosHashEntryWLock(pHashObj, pe);
 
   // double check after locked
   if (pe->num == 0) {
     assert(pe->next == NULL);
-    taosWUnLockLatch(&pe->latch);
 
-    __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
+    taosHashEntryWUnlock(pHashObj, pe);
+    taosHashRUnlock(pHashObj);
     return -1;
   }
 
-  int32_t        code = -1;
+  int code = -1;
   SHashNode *pNode = pe->next;
   SHashNode *prevNode = NULL;
 
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
-        pNode->removed == 0)
-      break;
+    if ((pNode->keyLen == keyLen) &&
+        ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
+        pNode->removed == 0) {
+      code = 0;  // it is found
 
-    prevNode = pNode;
-    pNode = pNode->next;
-  }
+      atomic_sub_fetch_32(&pNode->refCount, 1);
+      pNode->removed = 1;
+      if (pNode->refCount <= 0) {
+        if (prevNode == NULL) {
+          pe->next = pNode->next;
+        } else {
+          prevNode->next = pNode->next;
+        }
 
-  if (pNode) {
-    code = 0;  // it is found
+        if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
 
-    pNode->count--;
-    pNode->removed = 1;
-    if (pNode->count <= 0) {
-      if (prevNode) {
-        prevNode->next = pNode->next;
-      } else {
-        pe->next = pNode->next;
+        pe->num--;
+        atomic_sub_fetch_64(&pHashObj->size, 1);
+        FREE_HASH_NODE(pNode);
       }
-
-      //      if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
-
-      pe->num--;
-      atomic_sub_fetch_32(&pHashObj->size, 1);
-      FREE_HASH_NODE(pHashObj, pNode);
+    } else {
+      prevNode = pNode;
+      pNode = pNode->next;
     }
   }
 
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosWUnLockLatch(&pe->latch);
-  }
-
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashEntryWUnlock(pHashObj, pe);
+  taosHashRUnlock(pHashObj);
 
   return code;
 }
 
-int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) {
-  if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
-    return 0;
-  }
-
-  // disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
-
-  int32_t numOfEntries = (int32_t)pHashObj->capacity;
-  for (int32_t i = 0; i < numOfEntries; ++i) {
-    SHashEntry *pEntry = pHashObj->hashList[i];
-    if (pEntry->num == 0) {
-      continue;
-    }
-
-    if (pHashObj->type == HASH_ENTRY_LOCK) {
-      taosWLockLatch(&pEntry->latch);
-    }
-
-    // todo remove the first node
-    SHashNode *pNode = NULL;
-    while ((pNode = pEntry->next) != NULL) {
-      if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) {
-        pEntry->num -= 1;
-        atomic_sub_fetch_32(&pHashObj->size, 1);
-
-        pEntry->next = pNode->next;
-
-        if (pEntry->num == 0) {
-          assert(pEntry->next == NULL);
-        } else {
-          assert(pEntry->next != NULL);
-        }
-
-        FREE_HASH_NODE(pHashObj, pNode);
-      } else {
-        break;
-      }
-    }
-
-    // handle the following node
-    if (pNode != NULL) {
-      assert(pNode == pEntry->next);
-      SHashNode *pNext = NULL;
-
-      while ((pNext = pNode->next) != NULL) {
-        // not qualified, remove it
-        if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) {
-          pNode->next = pNext->next;
-          pEntry->num -= 1;
-          atomic_sub_fetch_32(&pHashObj->size, 1);
-
-          if (pEntry->num == 0) {
-            assert(pEntry->next == NULL);
-          } else {
-            assert(pEntry->next != NULL);
-          }
-
-          FREE_HASH_NODE(pHashObj, pNext);
-        } else {
-          pNode = pNext;
-        }
-      }
-    }
-
-    if (pHashObj->type == HASH_ENTRY_LOCK) {
-      taosWUnLockLatch(&pEntry->latch);
-    }
-  }
-
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
-  return 0;
+int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
+  return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0);
 }
 
 void taosHashClear(SHashObj *pHashObj) {
@@ -593,12 +536,12 @@ void taosHashClear(SHashObj *pHashObj) {
 
   SHashNode *pNode, *pNext;
 
-  __wr_lock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashWLock(pHashObj);
 
   for (int32_t i = 0; i < pHashObj->capacity; ++i) {
     SHashEntry *pEntry = pHashObj->hashList[i];
     if (pEntry->num == 0) {
-      assert(pEntry->next == 0);
+      assert(pEntry->next == NULL);
       continue;
     }
 
@@ -607,7 +550,7 @@ void taosHashClear(SHashObj *pHashObj) {
 
     while (pNode) {
       pNext = pNode->next;
-      FREE_HASH_NODE(pHashObj, pNode);
+      FREE_HASH_NODE(pNode);
 
       pNode = pNext;
     }
@@ -616,10 +559,11 @@ void taosHashClear(SHashObj *pHashObj) {
     pEntry->next = NULL;
   }
 
-  atomic_store_32(&pHashObj->size, 0);
-  __wr_unlock((void *)&pHashObj->lock, pHashObj->type);
+  pHashObj->size = 0;
+  taosHashWUnlock(pHashObj);
 }
 
+// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj)
 void taosHashCleanup(SHashObj *pHashObj) {
   if (pHashObj == NULL) {
     return;
@@ -636,26 +580,29 @@ void taosHashCleanup(SHashObj *pHashObj) {
   }
 
   taosArrayDestroy(pHashObj->pMemBlock);
-
-  memset(pHashObj, 0, sizeof(SHashObj));
   free(pHashObj);
 }
 
 // for profile only
-int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
+int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
   if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
     return 0;
   }
 
   int32_t num = 0;
 
+  taosHashRLock((SHashObj*) pHashObj);
   for (int32_t i = 0; i < pHashObj->size; ++i) {
     SHashEntry *pEntry = pHashObj->hashList[i];
+
+    // fine grain per entry lock is not held since this is used
+    // for profiling only and doesn't need an accurate count.
     if (num < pEntry->num) {
       num = pEntry->num;
     }
   }
 
+  taosHashRUnlock((SHashObj*) pHashObj);
   return num;
 }
 
@@ -664,28 +611,24 @@ void taosHashTableResize(SHashObj *pHashObj) {
     return;
   }
 
-  // double the original capacity
-  SHashNode *pNode = NULL;
-  SHashNode *pNext = NULL;
-
-  int32_t newSize = (int32_t)(pHashObj->capacity << 1u);
-  if (newSize > HASH_MAX_CAPACITY) {
-    //    uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached",
-    //           pHashObj->capacity, HASH_MAX_CAPACITY);
+  int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
+  if (newCapacity > HASH_MAX_CAPACITY) {
+//    uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
+//           pHashObj->capacity, HASH_MAX_CAPACITY);
     return;
   }
 
   int64_t st = taosGetTimestampUs();
-  void   *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize);
-  if (pNewEntryList == NULL) {  // todo handle error
-    //    uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity);
+  void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity);
+  if (pNewEntryList == NULL) {
+//    uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
     return;
   }
 
   pHashObj->hashList = pNewEntryList;
 
-  size_t inc = newSize - pHashObj->capacity;
-  void  *p = calloc(inc, sizeof(SHashEntry));
+  size_t inc = newCapacity - pHashObj->capacity;
+  void * p = calloc(inc, sizeof(SHashEntry));
 
   for (int32_t i = 0; i < inc; ++i) {
     pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
@@ -693,92 +636,62 @@ void taosHashTableResize(SHashObj *pHashObj) {
 
   taosArrayPush(pHashObj->pMemBlock, &p);
 
-  pHashObj->capacity = newSize;
-  for (int32_t i = 0; i < pHashObj->capacity; ++i) {
-    SHashEntry *pe = pHashObj->hashList[i];
-
-    if (pe->num == 0) {
-      assert(pe->next == NULL);
-    } else {
-      assert(pe->next != NULL);
-    }
+  pHashObj->capacity = newCapacity;
+  for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
+    SHashEntry *pe = pHashObj->hashList[idx];
+    SHashNode *pNode;
+    SHashNode *pNext;
+    SHashNode *pPrev = NULL;
 
     if (pe->num == 0) {
       assert(pe->next == NULL);
       continue;
     }
 
-    while ((pNode = pe->next) != NULL) {
-      int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
-      if (j != i) {
-        pe->num -= 1;
-        pe->next = pNode->next;
-
-        if (pe->num == 0) {
-          assert(pe->next == NULL);
-        } else {
-          assert(pe->next != NULL);
-        }
-
-        SHashEntry *pNewEntry = pHashObj->hashList[j];
-        pushfrontNodeInEntryList(pNewEntry, pNode);
-      } else {
-        break;
-      }
-    }
-
-    if (pNode != NULL) {
-      while ((pNext = pNode->next) != NULL) {
-        int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity);
-        if (j != i) {
-          pe->num -= 1;
-
-          pNode->next = pNext->next;
-          pNext->next = NULL;
-
-          // added into new slot
-          SHashEntry *pNewEntry = pHashObj->hashList[j];
+    pNode = pe->next;
 
-          if (pNewEntry->num == 0) {
-            assert(pNewEntry->next == NULL);
-          } else {
-            assert(pNewEntry->next != NULL);
-          }
+    assert(pNode != NULL);
 
-          pushfrontNodeInEntryList(pNewEntry, pNext);
+    while (pNode != NULL) {
+      int32_t newIdx = HASH_INDEX(pNode->hashVal, pHashObj->capacity);
+      pNext = pNode->next;
+      if (newIdx != idx) {
+        pe->num -= 1;
+        if (pPrev == NULL) {
+          pe->next = pNext;
         } else {
-          pNode = pNext;
+          pPrev->next = pNext;
         }
-      }
 
-      if (pe->num == 0) {
-        assert(pe->next == NULL);
+        SHashEntry *pNewEntry = pHashObj->hashList[newIdx];
+        pushfrontNodeInEntryList(pNewEntry, pNode);
       } else {
-        assert(pe->next != NULL);
+        pPrev = pNode;
       }
+      pNode = pNext;
     }
   }
 
   int64_t et = taosGetTimestampUs();
 
-  uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
-         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
+//  uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
+//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
 }
 
 SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
   SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize);
 
   if (pNewNode == NULL) {
-    uError("failed to allocate memory, reason:%s", strerror(errno));
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  pNewNode->keyLen = (uint32_t)keyLen;
+  pNewNode->keyLen  = (uint32_t)keyLen;
   pNewNode->hashVal = hashVal;
   pNewNode->dataLen = (uint32_t)dsize;
-  pNewNode->count = 1;
+  pNewNode->refCount= 1;
   pNewNode->removed = 0;
-  pNewNode->next = NULL;
+  pNewNode->next    = NULL;
 
   memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
   memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
@@ -800,51 +713,32 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
     return 0;
   }
 
-  return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) +
-         sizeof(SHashObj);
+  return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
 }
 
-FORCE_INLINE int32_t taosHashGetKey(void *data, void **key, size_t *keyLen) {
-  if (NULL == data || NULL == key) {
-    return -1;
-  }
-
-  SHashNode *node = GET_HASH_PNODE(data);
-  *key = GET_HASH_NODE_KEY(node);
-  if (keyLen) {
+void *taosHashGetKey(void *data, size_t* keyLen) {
+  SHashNode * node = GET_HASH_PNODE(data);
+  if (keyLen != NULL) {
     *keyLen = node->keyLen;
   }
 
-  return 0;
-}
-
-FORCE_INLINE int32_t taosHashGetDataLen(void *data) {
-  SHashNode *node = GET_HASH_PNODE(data);
-  return node->keyLen;
-}
-
-FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) {
-  SHashNode *node = GET_HASH_PNODE(data);
-  return node->keyLen;
+  return GET_HASH_NODE_KEY(node);
 }
 
 // release the pNode, return next pNode, and lock the current entry
-static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) {
+static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
   SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p);
   SHashNode *prevNode = NULL;
 
   *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[*slot];
 
-  // lock entry
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosWLockLatch(&pe->latch);
-  }
+  taosHashEntryWLock(pHashObj, pe);
 
   SHashNode *pNode = pe->next;
-
   while (pNode) {
-    if (pNode == pOld) break;
+    if (pNode == pOld)
+      break;
 
     prevNode = pNode;
     pNode = pNode->next;
@@ -857,8 +751,8 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) {
       pNode = pNode->next;
     }
 
-    pOld->count--;
-    if (pOld->count <= 0) {
+    atomic_sub_fetch_32(&pOld->refCount, 1);
+    if (pOld->refCount <=0) {
       if (prevNode) {
         prevNode->next = pOld->next;
       } else {
@@ -866,11 +760,11 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) {
       }
 
       pe->num--;
-      atomic_sub_fetch_32(&pHashObj->size, 1);
-      FREE_HASH_NODE(pHashObj, pOld);
+      atomic_sub_fetch_64(&pHashObj->size, 1);
+      FREE_HASH_NODE(pOld);
     }
   } else {
-    uError("pNode:%p data:%p is not there!!!", pNode, p);
+//    uError("pNode:%p data:%p is not there!!!", pNode, p);
   }
 
   return pNode;
@@ -879,20 +773,18 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) {
 void *taosHashIterate(SHashObj *pHashObj, void *p) {
   if (pHashObj == NULL) return NULL;
 
-  int32_t   slot = 0;
+  int  slot = 0;
   char *data = NULL;
 
   // only add the read lock to disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashRLock(pHashObj);
 
   SHashNode *pNode = NULL;
   if (p) {
     pNode = taosHashReleaseNode(pHashObj, p, &slot);
     if (pNode == NULL) {
       SHashEntry *pe = pHashObj->hashList[slot];
-      if (pHashObj->type == HASH_ENTRY_LOCK) {
-        taosWUnLockLatch(&pe->latch);
-      }
+      taosHashEntryWUnlock(pHashObj, pe);
 
       slot = slot + 1;
     }
@@ -902,10 +794,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) {
     for (; slot < pHashObj->capacity; ++slot) {
       SHashEntry *pe = pHashObj->hashList[slot];
 
-      // lock entry
-      if (pHashObj->type == HASH_ENTRY_LOCK) {
-        taosWLockLatch(&pe->latch);
-      }
+      taosHashEntryWLock(pHashObj, pe);
 
       pNode = pe->next;
       while (pNode) {
@@ -915,23 +804,22 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) {
 
       if (pNode) break;
 
-      if (pHashObj->type == HASH_ENTRY_LOCK) {
-        taosWUnLockLatch(&pe->latch);
-      }
+      taosHashEntryWUnlock(pHashObj, pe);
     }
   }
 
   if (pNode) {
     SHashEntry *pe = pHashObj->hashList[slot];
 
-    uint16_t prevRef = atomic_load_16(&pNode->count);
-    uint16_t afterRef = atomic_add_fetch_16(&pNode->count, 1);
+    uint16_t prevRef = atomic_load_16(&pNode->refCount);
+    uint16_t afterRef = atomic_add_fetch_16(&pNode->refCount, 1);
+    ASSERT(prevRef < afterRef);
 
     // the reference count value is overflow, which will cause the delete node operation immediately.
     if (prevRef > afterRef) {
       uError("hash entry ref count overflow, prev ref:%d, current ref:%d", prevRef, afterRef);
       // restore the value
-      atomic_sub_fetch_16(&pNode->count, 1);
+      atomic_sub_fetch_16(&pNode->refCount, 1);
       data = NULL;
     } else {
       data = GET_HASH_NODE_DATA(pNode);
@@ -941,12 +829,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) {
       uWarn("hash entry ref count is abnormally high: %d", afterRef);
     }
 
-    if (pHashObj->type == HASH_ENTRY_LOCK) {
-      taosWUnLockLatch(&pe->latch);
-    }
+    taosHashEntryWUnlock(pHashObj, pe);
   }
 
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashRUnlock(pHashObj);
   return data;
 }
 
@@ -954,17 +840,21 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
   if (pHashObj == NULL || p == NULL) return;
 
   // only add the read lock to disable the resize process
-  __rd_lock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashRLock(pHashObj);
 
-  int32_t slot;
+  int slot;
   taosHashReleaseNode(pHashObj, p, &slot);
 
   SHashEntry *pe = pHashObj->hashList[slot];
-  if (pHashObj->type == HASH_ENTRY_LOCK) {
-    taosWUnLockLatch(&pe->latch);
-  }
 
-  __rd_unlock((void *)&pHashObj->lock, pHashObj->type);
+  taosHashEntryWUnlock(pHashObj, pe);
+  taosHashRUnlock(pHashObj);
+}
+
+//TODO remove it
+void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
+  void* p = NULL;
+  return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true);
 }
 
 void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); }

source/util/src/tpagedbuf.c

@@ -42,8 +42,8 @@ struct SDiskbasedBuf {
   bool      comp;              // compressed before flushed to disk
   uint64_t  nextPos;           // next page flush position
 
-  uint64_t            qId;          // for debug purpose
-  bool                printStatis;  // Print statistics info when closing this buffer.
+  char*     id;          // for debug purpose
+  bool      printStatis;  // Print statistics info when closing this buffer.
   SDiskbasedBufStatis statis;
 };
 
@@ -269,11 +269,12 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag
   SPageInfo* ppi = malloc(sizeof(SPageInfo));
 
   ppi->pageId = pageId;
-  ppi->pData = NULL;
+  ppi->pData  = NULL;
   ppi->offset = -1;
   ppi->length = -1;
-  ppi->used = true;
-  ppi->pn = NULL;
+  ppi->used   = true;
+  ppi->pn     = NULL;
+  ppi->dirty  = false;
 
   return *(SPageInfo**)taosArrayPush(list, &ppi);
 }
@@ -356,7 +357,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) {
   return ppi;
 }
 
-int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
+int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id,
                            const char* dir) {
   *pBuf = calloc(1, sizeof(SDiskbasedBuf));
 
@@ -366,13 +367,13 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
   }
 
   pPBuf->pageSize = pagesize;
-  pPBuf->numOfPages = 0;  // all pages are in buffer in the first place
+  pPBuf->numOfPages   = 0;  // all pages are in buffer in the first place
   pPBuf->totalBufSize = 0;
   pPBuf->inMemPages = inMemBufSize / pagesize;  // maximum allowed pages, it is a soft limit.
   pPBuf->allocateId = -1;
-  pPBuf->comp = true;
-  pPBuf->pFile = NULL;
-  pPBuf->qId = qId;
+  pPBuf->comp     = true;
+  pPBuf->pFile    = NULL;
+  pPBuf->id       = strdup(id);
   pPBuf->fileSize = 0;
   pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem));
   pPBuf->freePgList = tdListNew(POINTER_BYTES);
@@ -471,7 +472,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
 
     return (void*)(GET_DATA_PAYLOAD(*pi));
   } else {  // not in memory
-    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0);
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
 
     char* availablePage = NULL;
     if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
@@ -493,9 +494,12 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     lruListPushFront(pBuf->lruList, *pi);
     (*pi)->used = true;
 
-    int32_t code = loadPageFromDisk(pBuf, *pi);
-    if (code != 0) {
-      return NULL;
+    // some data has been flushed to disk, and needs to be loaded into buffer again.
+    if ((*pi)->length > 0 && (*pi)->offset >= 0) {
+      int32_t code = loadPageFromDisk(pBuf, *pi);
+      if (code != 0) {
+        return NULL;
+      }
     }
 
     return (void*)(GET_DATA_PAYLOAD(*pi));
@@ -540,13 +544,13 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
   if (pBuf->pFile != NULL) {
     uDebug(
         "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
-        "size:%.2f Kb, %" PRIx64 "\n",
+        "size:%.2f Kb, %s\n",
         pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
-        listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
+        listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id);
 
     taosCloseFile(&pBuf->pFile);
   } else {
-    uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId);
+    uDebug("Paged buffer closed, total:%.2f Kb, no file created, %s", pBuf->totalBufSize / 1024.0, pBuf->id);
   }
 
   // print the statistics information
@@ -584,6 +588,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
   taosHashCleanup(pBuf->groupSet);
   taosHashCleanup(pBuf->all);
 
+  tfree(pBuf->id);
   tfree(pBuf->assistBuf);
   tfree(pBuf);
 }
@@ -639,9 +644,9 @@ void dBufPrintStatis(const SDiskbasedBuf* pBuf) {
 
   printf(
       "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f "
-      "Kb, %" PRIx64 "\n",
+      "Kb, %s\n",
       pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
-      listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
+      listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id);
 
   printf(
       "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n",

source/util/test/arrayTest.cpp

@@ -43,6 +43,9 @@ static void remove_batch_test() {
   taosArrayPush(delList, &a);
   taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList));
   EXPECT_EQ(taosArrayGetSize(pa), 17);
+
+  taosArrayDestroy(pa);
+  taosArrayDestroy(delList);
 }
 }  // namespace
 
@@ -79,4 +82,6 @@ TEST(arrayTest, array_search_test) {
     }
     
   }
+
+  taosArrayDestroy(pa);
 }

source/util/test/cacheTest.cpp

@@ -6,7 +6,7 @@
 #include "tcache.h"
 
 // test cache
-TEST(testCase, client_cache_test) {
+TEST(cacheTest, client_cache_test) {
   const int32_t REFRESH_TIME_IN_SEC = 2;
   SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test");
 
@@ -92,7 +92,7 @@ TEST(testCase, client_cache_test) {
   taosCacheCleanup(tscMetaCache);
 }
 
-TEST(testCase, cache_resize_test) {
+TEST(cacheTest, cache_iter_test) {
   const int32_t REFRESH_TIME_IN_SEC = 2;
   auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL, "test");
 
@@ -107,6 +107,7 @@ TEST(testCase, cache_resize_test) {
     int32_t len = sprintf(key, "abc_%7d", i);
     taosCachePut(pCache, key, strlen(key), data, len, 3600);
   }
+
   uint64_t endTime = taosGetTimestampUs();
 
   printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num);
@@ -120,5 +121,22 @@ TEST(testCase, cache_resize_test) {
   endTime = taosGetTimestampUs();
   printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num);
 
+  int32_t count = 0;
+  SCacheIter* pIter = taosCacheCreateIter(pCache);
+  while(taosCacheIterNext(pIter)) {
+    size_t keyLen = 0;
+    size_t dataLen = 0;
+
+    char* key1 = static_cast<char*>(taosCacheIterGetKey(pIter, &keyLen));
+    char* data1 = static_cast<char*>(taosCacheIterGetData(pIter, &dataLen));
+
+//    char d[256] = {0};
+//    memcpy(d, data1, dataLen);
+//    char k[256] = {0};
+//    memcpy(k, key1, keyLen);
+  }
+
+  ASSERT_EQ(count, num);
+
   taosCacheCleanup(pCache);
-}
\ No newline at end of file
+}

source/util/test/encodeTest.cpp

@@ -201,8 +201,8 @@ TEST(td_encode_test, encode_decode_cstr) {
     }
   }
 
-  delete buf;
-  delete cstr;
+  delete[] buf;
+  delete[] cstr;
 }
 
 typedef struct {
@@ -354,7 +354,7 @@ static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) {
   tEndDecode(pCoder);
   return 0;
 }
-
+#if 0
 TEST(td_encode_test, compound_struct_encode_test) {
   SCoder       encoder, decoder;
   uint8_t *    buf1;
@@ -436,5 +436,5 @@ TEST(td_encode_test, compound_struct_encode_test) {
   GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b);
   tCoderClear(&decoder);
 }
-
+#endif
 #pragma GCC diagnostic pop
\ No newline at end of file

source/util/test/hashTest.cpp

@@ -106,7 +106,7 @@ void noLockPerformanceTest() {
   ASSERT_EQ(taosHashGetSize(hashTable), 0);
   
   char key[128] = {0};
-  int32_t num = 5000000;
+  int32_t num = 5000;
   
   int64_t st = taosGetTimestampUs();
   
@@ -186,10 +186,15 @@ void acquireRleaseTest() {
 
   printf("%s,expect:%s", pdata->p, str3);
   ASSERT_TRUE(strcmp(pdata->p, str3) == 0);
-  
+
+  tfree(pdata->p);
+
   taosHashRelease(hashTable, pdata);
   num = taosHashGetSize(hashTable);
   ASSERT_EQ(num, 1);
+
+  taosHashCleanup(hashTable);
+  tfree(data.p);
 }
 
 }

source/util/test/pageBufferTest.cpp

@@ -12,145 +12,150 @@
 namespace {
 // simple test
 void simpleTest() {
-  SDiskbasedBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/");
+  SDiskbasedBuf* pBuf = NULL;
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/");
   
   int32_t pageId = 0;
   int32_t groupId = 0;
   
-  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
+  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
   ASSERT_TRUE(pBufPage != NULL);
   
-  ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
+  ASSERT_EQ(getTotalBufSize(pBuf), 1024);
   
-  SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
+  SIDList list = getDataBufPagesIdList(pBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(list), 1);
-  ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1);
+  ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
 
-  releaseBufPage(pResultBuf, pBufPage);
+  releaseBufPage(pBuf, pBufPage);
 
-  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
+  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
 
-  SFilePage* t = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* t = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t == pBufPage1);
 
-  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t1 == pBufPage2);
 
-  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t2 == pBufPage3);
 
-  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t3 == pBufPage4);
 
-  SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  releaseBufPage(pBuf, pBufPage2);
+
+  SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t4 == pBufPage5);
 
-  destroyDiskbasedBuf(pResultBuf);
+  destroyDiskbasedBuf(pBuf);
 }
 
 void writeDownTest() {
-  SDiskbasedBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
+  SDiskbasedBuf* pBuf = NULL;
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
 
   int32_t pageId = 0;
   int32_t writePageId = 0;
   int32_t groupId = 0;
   int32_t nx = 12345;
 
-  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
+  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
   ASSERT_TRUE(pBufPage != NULL);
 
   *(int32_t*)(pBufPage->data) = nx;
   writePageId = pageId;
-  releaseBufPage(pResultBuf, pBufPage);
+  
+  setBufPageDirty(pBufPage, true);
+  releaseBufPage(pBuf, pBufPage);
 
-  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t1 == pBufPage1);
   ASSERT_TRUE(pageId == 1);
 
-  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t2 == pBufPage2);
   ASSERT_TRUE(pageId == 2);
 
-  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t3 == pBufPage3);
   ASSERT_TRUE(pageId == 3);
 
-  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t4 == pBufPage4);
   ASSERT_TRUE(pageId == 4);
-  releaseBufPage(pResultBuf, t4);
+  releaseBufPage(pBuf, t4);
 
   // flush the written page to disk, and read it out again
-  SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pResultBuf, writePageId));
+  SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pBuf, writePageId));
   ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
 
-  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  SArray* pa = getDataBufPagesIdList(pBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(pa), 5);
 
-  destroyDiskbasedBuf(pResultBuf);
+  destroyDiskbasedBuf(pBuf);
 }
 
 void recyclePageTest() {
-  SDiskbasedBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
+  SDiskbasedBuf* pBuf = NULL;
+  int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
 
   int32_t pageId = 0;
   int32_t writePageId = 0;
   int32_t groupId = 0;
   int32_t nx = 12345;
 
-  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
+  SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
   ASSERT_TRUE(pBufPage != NULL);
-  releaseBufPage(pResultBuf, pBufPage);
+  releaseBufPage(pBuf, pBufPage);
 
-  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t1 == pBufPage1);
   ASSERT_TRUE(pageId == 1);
 
-  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t2 == pBufPage2);
   ASSERT_TRUE(pageId == 2);
 
-  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t3 == pBufPage3);
   ASSERT_TRUE(pageId == 3);
 
-  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t4 == pBufPage4);
   ASSERT_TRUE(pageId == 4);
-  releaseBufPage(pResultBuf, t4);
+  releaseBufPage(pBuf, t4);
 
-  SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
-  SFilePage* t5 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
+  SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
+  SFilePage* t5 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
   ASSERT_TRUE(t5 == pBufPage5);
   ASSERT_TRUE(pageId == 5);
+  releaseBufPage(pBuf, t5);
 
   // flush the written page to disk, and read it out again
-  SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pResultBuf, writePageId));
+  SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pBuf, writePageId));
   *(int32_t*)(pBufPagex->data) = nx;
   writePageId = pageId;   // update the data
-  releaseBufPage(pResultBuf, pBufPagex);
+  releaseBufPage(pBuf, pBufPagex);
 
-  SFilePage* pBufPagex1 = static_cast<SFilePage*>(getBufPage(pResultBuf, 1));
+  SFilePage* pBufPagex1 = static_cast<SFilePage*>(getBufPage(pBuf, 1));
 
-  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  SArray* pa = getDataBufPagesIdList(pBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(pa), 6);
 
-  destroyDiskbasedBuf(pResultBuf);
+  destroyDiskbasedBuf(pBuf);
 }
 } // namespace
 

tests @ 904e6f0e

-Subproject commit 08ed39f0a5fcbbfb5a630b945ab3d1998d4b4136
+Subproject commit 904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c