Merge pull request #12802 from taosdata/feature/stream

fix: memory error

cmake/cmake.define

@@ -71,8 +71,8 @@ ELSE ()
     ENDIF ()
 
     IF (${SANITIZER} MATCHES "true")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")

include/common/tmsg.h

@@ -2524,7 +2524,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
   buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
   buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
   pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
-  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
   pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
   if (pRsp->blockNum != 0) {

source/util/src/thash.c

@@ -15,8 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "thash.h"
-#include "taoserror.h"
 #include "os.h"
+#include "taoserror.h"
 #include "tlog.h"
 
 // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT
@@ -27,13 +27,13 @@
 
 #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
 
-#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
-#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
-#define GET_HASH_PNODE(_n)     ((SHashNode *)((char*)(_n) - sizeof(SHashNode)))
+#define GET_HASH_NODE_KEY(_n)  ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
+#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
+#define GET_HASH_PNODE(_n)     ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
 
 #define FREE_HASH_NODE(_fp, _n) \
   do {                          \
-/*    if (_fp != NULL) {          \
+    /*    if (_fp != NULL) {    \
           (_fp)(_n);            \
         }*/                     \
     taosMemoryFreeClear(_n);    \
@@ -56,7 +56,7 @@ typedef struct SHashEntry {
 } SHashEntry;
 
 struct SHashObj {
-  SHashEntry **     hashList;
+  SHashEntry      **hashList;
   size_t            capacity;      // number of slots
   int64_t           size;          // number of elements in hash table
   _hash_fn_t        hashFp;        // hash function
@@ -65,7 +65,7 @@ struct SHashObj {
   SRWLatch          lock;          // read-write spin lock
   SHashLockTypeE    type;          // lock type
   bool              enableUpdate;  // enable update
-  SArray *          pMemBlock;     // memory block allocated for SHashEntry
+  SArray           *pMemBlock;     // memory block allocated for SHashEntry
   _hash_before_fn_t callbackFp;    // function invoked before return the value to caller
 };
 
@@ -103,14 +103,14 @@ static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) {
   taosRUnLockLatch(&pHashObj->lock);
 }
 
-static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) {
+static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry *pe) {
   if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
   taosWLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
+static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
   if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
@@ -118,7 +118,7 @@ static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEnt
   taosWUnLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) {
+static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry *pe) {
   if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
@@ -126,7 +126,7 @@ static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry
   taosRLockLatch(&pe->latch);
 }
 
-static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
+static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
   if (pHashObj->type == HASH_NO_LOCK) {
     return;
   }
@@ -142,12 +142,11 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
   return i;
 }
 
-static FORCE_INLINE SHashNode *
-doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
+static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen,
+                                                   uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) &&
-        ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
+    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
         pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
@@ -186,7 +185,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
  * @param pNode      the old node with requested key
  * @param pNewNode   the new node with requested key
  */
-static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode,
+                                          SHashNode *pNewNode) {
   assert(pNode->keyLen == pNewNode->keyLen);
 
   atomic_sub_fetch_16(&pNode->refCount, 1);
@@ -227,9 +227,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj);
  * @param pHashObj
  * @return
  */
-static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) {
-  return taosHashGetSize(pHashObj) == 0;
-}
+static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; }
 
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
   if (fn == NULL) {
@@ -305,7 +303,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
   if (pHashObj == NULL) {
     return 0;
   }
-  return (int32_t)atomic_load_64((int64_t*)&pHashObj->size);
+  return (int32_t)atomic_load_64((int64_t *)&pHashObj->size);
 }
 
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
@@ -340,10 +338,9 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
   }
 #endif
 
-  SHashNode* prev = NULL;
+  SHashNode *prev = NULL;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) &&
-        (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
+    if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
         pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
@@ -391,27 +388,27 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
   }
 }
 
-static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef);
+static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef);
 
 void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
-  void* p = NULL;
+  void *p = NULL;
   return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false);
 }
 
 int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) {
   terrno = 0;
-  /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
+  /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
   return terrno;
 }
 
-int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) {
+int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t *size) {
   terrno = 0;
 
-  /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
+  /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
   return terrno;
 }
 
-void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) {
+void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef) {
   if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
     return NULL;
   }
@@ -457,7 +454,7 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void**
         }
       } else if (*size < pNode->dataLen) {
         *size = pNode->dataLen;
-        char* tmp = taosMemoryRealloc(*d, *size);
+        char *tmp = taosMemoryRealloc(*d, *size);
         if (tmp == NULL) {
           terrno = TSDB_CODE_OUT_OF_MEMORY;
           return NULL;
@@ -513,8 +510,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
   SHashNode *prevNode = NULL;
 
   while (pNode) {
-    if ((pNode->keyLen == keyLen) &&
-        ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
+    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
         pNode->removed == 0) {
       code = 0;  // it is found
 
@@ -598,14 +594,14 @@ void taosHashCleanup(SHashObj *pHashObj) {
 }
 
 // for profile only
-int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
+int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
   if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
     return 0;
   }
 
   int32_t num = 0;
 
-  taosHashRLock((SHashObj*) pHashObj);
+  taosHashRLock((SHashObj *)pHashObj);
   for (int32_t i = 0; i < pHashObj->size; ++i) {
     SHashEntry *pEntry = pHashObj->hashList[i];
 
@@ -616,7 +612,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
     }
   }
 
-  taosHashRUnlock((SHashObj*) pHashObj);
+  taosHashRUnlock((SHashObj *)pHashObj);
   return num;
 }
 
@@ -627,22 +623,22 @@ void taosHashTableResize(SHashObj *pHashObj) {
 
   int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
   if (newCapacity > HASH_MAX_CAPACITY) {
-//    uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
-//           pHashObj->capacity, HASH_MAX_CAPACITY);
+    //    uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
+    //           pHashObj->capacity, HASH_MAX_CAPACITY);
     return;
   }
 
   int64_t st = taosGetTimestampUs();
   void   *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
   if (pNewEntryList == NULL) {
-//    uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
+    //    uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
     return;
   }
 
   pHashObj->hashList = pNewEntryList;
 
   size_t inc = newCapacity - pHashObj->capacity;
-  void * p = taosMemoryCalloc(inc, sizeof(SHashEntry));
+  void  *p = taosMemoryCalloc(inc, sizeof(SHashEntry));
 
   for (int32_t i = 0; i < inc; ++i) {
     pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
@@ -688,12 +684,13 @@ void taosHashTableResize(SHashObj *pHashObj) {
 
   int64_t et = taosGetTimestampUs();
 
-//  uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
-//         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
+  //  uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms",
+  //  (int32_t)pHashObj->capacity,
+  //         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
 }
 
 SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
-  SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize);
+  SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize + 1);
 
   if (pNewNode == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -703,7 +700,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
   pNewNode->keyLen = (uint32_t)keyLen;
   pNewNode->hashVal = hashVal;
   pNewNode->dataLen = (uint32_t)dsize;
-  pNewNode->refCount= 1;
+  pNewNode->refCount = 1;
   pNewNode->removed = 0;
   pNewNode->next = NULL;
 
@@ -727,11 +724,12 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
     return 0;
   }
 
-  return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
+  return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void *))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) +
+         sizeof(SHashObj);
 }
 
-void *taosHashGetKey(void *data, size_t* keyLen) {
-  SHashNode * node = GET_HASH_PNODE(data);
+void *taosHashGetKey(void *data, size_t *keyLen) {
+  SHashNode *node = GET_HASH_PNODE(data);
   if (keyLen != NULL) {
     *keyLen = node->keyLen;
   }
@@ -751,8 +749,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
 
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if (pNode == pOld)
-      break;
+    if (pNode == pOld) break;
 
     prevNode = pNode;
     pNode = pNode->next;
@@ -766,7 +763,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
     }
 
     atomic_sub_fetch_16(&pOld->refCount, 1);
-    if (pOld->refCount <=0) {
+    if (pOld->refCount <= 0) {
       if (prevNode) {
         prevNode->next = pOld->next;
       } else {
@@ -778,7 +775,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
       FREE_HASH_NODE(pHashObj->freeFp, pOld);
     }
   } else {
-//    uError("pNode:%p data:%p is not there!!!", pNode, p);
+    //    uError("pNode:%p data:%p is not there!!!", pNode, p);
   }
 
   return pNode;
@@ -865,9 +862,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
   taosHashRUnlock(pHashObj);
 }
 
-//TODO remove it
+// TODO remove it
 void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
-  void* p = NULL;
+  void *p = NULL;
   return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true);
 }