Skip to content

T
TDengine

taosdata / TDengine
大约 1 年 前同步成功

通知 1184
Star 22015
Fork 4786
T
TDengine

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 fb6bebe6 编写于 作者: L Liu Jicong 提交者： GitHub
浏览文件
操作

Merge pull request #12802 from taosdata/feature/stream 

fix: memory error
上级 e00a9fa9 371717ee
展开全部 隐藏空白更改
内联 并排
Showing with 638 addition and 653 deletion
cmake/cmake.define
浏览文件 @ fb6bebe6
......@@ -71,8 +71,8 @@ ELSE ()
ENDIF ()
IF (${SANITIZER} MATCHES "true")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
MESSAGE(STATUS "Will compile with Address Sanitizer!")
ELSE ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
......
include/common/tmsg.h
浏览文件 @ fb6bebe6
......@@ -1352,7 +1352,7 @@ typedef struct {
typedef struct {
int32_t code;
char tbFName[TSDB_TABLE_FNAME_LEN];
char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t sversion;
int32_t tversion;
} SResReadyRsp;
......@@ -2524,7 +2524,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
if (pRsp->blockNum != 0) {
......
source/libs/catalog/src/catalog.c
浏览文件 @ fb6bebe6
此差异已折叠。
source/util/src/thash.c
浏览文件 @ fb6bebe6
......@@ -15,8 +15,8 @@
#define _DEFAULT_SOURCE
#include "thash.h"
#include "taoserror.h"
#include "os.h"
#include "taoserror.h"
#include "tlog.h"
// the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT
......@@ -27,36 +27,36 @@
#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode)))
#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
#define FREE_HASH_NODE(_fp, _n) \
do { \
/* if (_fp != NULL) { \
(_fp)(_n); \
}*/ \
/* if (_fp != NULL) { \
(_fp)(_n); \
}*/ \
taosMemoryFreeClear(_n); \
} while (0);
struct SHashNode {
SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t refCount; // reference count
int8_t removed; // flag to indicate removed
char data[];
SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t refCount; // reference count
int8_t removed; // flag to indicate removed
char data[];
};
typedef struct SHashEntry {
int32_t num; // number of elements in current entry
SRWLatch latch; // entry latch
SHashNode *next;
int32_t num; // number of elements in current entry
SRWLatch latch; // entry latch
SHashNode *next;
} SHashEntry;
struct SHashObj {
SHashEntry ** hashList;
SHashEntry **hashList;
size_t capacity; // number of slots
int64_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
......@@ -65,7 +65,7 @@ struct SHashObj {
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
bool enableUpdate; // enable update
SArray * pMemBlock; // memory block allocated for SHashEntry
SArray *pMemBlock; // memory block allocated for SHashEntry
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
};
......@@ -103,14 +103,14 @@ static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) {
taosRUnLockLatch(&pHashObj->lock);
}
static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) {
static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) {
return;
}
taosWLockLatch(&pe->latch);
}
static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) {
return;
}
......@@ -118,7 +118,7 @@ static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEnt
taosWUnLockLatch(&pe->latch);
}
static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) {
static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) {
return;
}
......@@ -126,7 +126,7 @@ static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry
taosRLockLatch(&pe->latch);
}
static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) {
static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry *pe) {
if (pHashObj->type == HASH_NO_LOCK) {
return;
}
......@@ -142,12 +142,11 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
return i;
}
static FORCE_INLINE SHashNode *
doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen,
uint32_t hashVal) {
SHashNode *pNode = pe->next;
while (pNode) {
if ((pNode->keyLen == keyLen) &&
((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
......@@ -186,7 +185,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param pNode the old node with requested key
* @param pNewNode the new node with requested key
*/
static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode,
SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen);
atomic_sub_fetch_16(&pNode->refCount, 1);
......@@ -227,9 +227,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj);
* @param pHashObj
* @return
*/
static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) {
return taosHashGetSize(pHashObj) == 0;
}
static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; }
SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
if (fn == NULL) {
......@@ -251,7 +249,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
pHashObj->capacity = taosHashCapacity((int32_t)capacity);
pHashObj->equalFp = memcmp;
pHashObj->hashFp = fn;
pHashObj->hashFp = fn;
pHashObj->type = type;
pHashObj->enableUpdate = update;
......@@ -305,7 +303,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
}
return (int32_t)atomic_load_64((int64_t*)&pHashObj->size);
return (int32_t)atomic_load_64((int64_t *)&pHashObj->size);
}
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
......@@ -340,10 +338,9 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
}
#endif
SHashNode* prev = NULL;
SHashNode *prev = NULL;
while (pNode) {
if ((pNode->keyLen == keyLen) &&
(*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 &&
pNode->removed == 0) {
assert(pNode->hashVal == hashVal);
break;
......@@ -391,27 +388,27 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
}
}
static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef);
static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef);
void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
void* p = NULL;
void *p = NULL;
return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false);
}
int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) {
terrno = 0;
/*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
/*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false);
return terrno;
}
int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) {
int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t *size) {
terrno = 0;
/*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
/*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false);
return terrno;
}
void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) {
void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
return NULL;
}
......@@ -449,15 +446,15 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void**
if (size != NULL) {
if (*d == NULL) {
*size = pNode->dataLen;
*size = pNode->dataLen;
*d = taosMemoryCalloc(1, *size);
if (*d == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
} else if (*size < pNode->dataLen) {
*size = pNode->dataLen;
char* tmp = taosMemoryRealloc(*d, *size);
*size = pNode->dataLen;
char *tmp = taosMemoryRealloc(*d, *size);
if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
......@@ -508,13 +505,12 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
return -1;
}
int code = -1;
int code = -1;
SHashNode *pNode = pe->next;
SHashNode *prevNode = NULL;
while (pNode) {
if ((pNode->keyLen == keyLen) &&
((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
pNode->removed == 0) {
code = 0; // it is found
......@@ -598,14 +594,14 @@ void taosHashCleanup(SHashObj *pHashObj) {
}
// for profile only
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) {
return 0;
}
int32_t num = 0;
taosHashRLock((SHashObj*) pHashObj);
taosHashRLock((SHashObj *)pHashObj);
for (int32_t i = 0; i < pHashObj->size; ++i) {
SHashEntry *pEntry = pHashObj->hashList[i];
......@@ -616,7 +612,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){
}
}
taosHashRUnlock((SHashObj*) pHashObj);
taosHashRUnlock((SHashObj *)pHashObj);
return num;
}
......@@ -627,22 +623,22 @@ void taosHashTableResize(SHashObj *pHashObj) {
int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
if (newCapacity > HASH_MAX_CAPACITY) {
// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
// pHashObj->capacity, HASH_MAX_CAPACITY);
// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
// pHashObj->capacity, HASH_MAX_CAPACITY);
return;
}
int64_t st = taosGetTimestampUs();
void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
if (pNewEntryList == NULL) {
// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
return;
}
pHashObj->hashList = pNewEntryList;
size_t inc = newCapacity - pHashObj->capacity;
void * p = taosMemoryCalloc(inc, sizeof(SHashEntry));
void *p = taosMemoryCalloc(inc, sizeof(SHashEntry));
for (int32_t i = 0; i < inc; ++i) {
pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
......@@ -653,9 +649,9 @@ void taosHashTableResize(SHashObj *pHashObj) {
pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
SHashEntry *pe = pHashObj->hashList[idx];
SHashNode *pNode;
SHashNode *pNext;
SHashNode *pPrev = NULL;
SHashNode *pNode;
SHashNode *pNext;
SHashNode *pPrev = NULL;
if (pe->num == 0) {
assert(pe->next == NULL);
......@@ -688,24 +684,25 @@ void taosHashTableResize(SHashObj *pHashObj) {
int64_t et = taosGetTimestampUs();
// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms",
// (int32_t)pHashObj->capacity,
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
}
SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize);
SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize + 1);
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->keyLen = (uint32_t)keyLen;
pNewNode->hashVal = hashVal;
pNewNode->dataLen = (uint32_t)dsize;
pNewNode->refCount= 1;
pNewNode->refCount = 1;
pNewNode->removed = 0;
pNewNode->next = NULL;
pNewNode->next = NULL;
memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
......@@ -727,11 +724,12 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return 0;
}
return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void *))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) +
sizeof(SHashObj);
}
void *taosHashGetKey(void *data, size_t* keyLen) {
SHashNode * node = GET_HASH_PNODE(data);
void *taosHashGetKey(void *data, size_t *keyLen) {
SHashNode *node = GET_HASH_PNODE(data);
if (keyLen != NULL) {
*keyLen = node->keyLen;
}
......@@ -751,8 +749,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
SHashNode *pNode = pe->next;
while (pNode) {
if (pNode == pOld)
break;
if (pNode == pOld) break;
prevNode = pNode;
pNode = pNode->next;
......@@ -766,7 +763,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
}
atomic_sub_fetch_16(&pOld->refCount, 1);
if (pOld->refCount <=0) {
if (pOld->refCount <= 0) {
if (prevNode) {
prevNode->next = pOld->next;
} else {
......@@ -778,7 +775,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
FREE_HASH_NODE(pHashObj->freeFp, pOld);
}
} else {
// uError("pNode:%p data:%p is not there!!!", pNode, p);
// uError("pNode:%p data:%p is not there!!!", pNode, p);
}
return pNode;
......@@ -787,7 +784,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
void *taosHashIterate(SHashObj *pHashObj, void *p) {
if (pHashObj == NULL) return NULL;
int slot = 0;
int slot = 0;
char *data = NULL;
// only add the read lock to disable the resize process
......@@ -865,9 +862,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
taosHashRUnlock(pHashObj);
}
//TODO remove it
// TODO remove it
void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
void* p = NULL;
void *p = NULL;
return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true);
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或